Merge pull request #1310 from jemrobinson/1304-allow-sres-without-dat…

…abases Allow flexible number of databases to be specified
alan-turing-institute · Oct 6, 2022 · 0c51dff · 0c51dff
2 parents 19ec094 + 8a2630f
commit 0c51dff
Show file tree

Hide file tree

Showing 14 changed files with 184 additions and 148 deletions.
diff --git a/deployment/common/Configuration.psm1 b/deployment/common/Configuration.psm1
@@ -1059,7 +1059,9 @@ function Get-SreConfig {
     # Databases
     # ---------
     $config.sre.databases = [ordered]@{
-        rg = "$($config.sre.rgPrefix)_DATABASES".ToUpper()
+        rg        = "$($config.sre.rgPrefix)_DATABASES".ToUpper()
+        enabled   = $False
+        instances = @()
     }
     $dbConfig = @{
         MSSQL      = @{port = "1433"; prefix = "MSSQL"; sku = "sqldev-gen2" }
@@ -1070,7 +1072,8 @@ function Get-SreConfig {
         if (-not @($dbConfig.Keys).Contains($databaseType)) {
             Add-LogMessage -Level Fatal "Database type '$databaseType' was not recognised!"
         }
-        $config.sre.databases["db$($databaseType.ToLower())"] = [ordered]@{
+        $databasePorts += $dbConfig[$databaseType].port
+        $config.sre.databases.instances += [ordered]@{
             adminPasswordSecretName   = "$($config.sre.shortName)-vm-admin-password-$($databaseType.ToLower())"
             dbAdminUsernameSecretName = "$($config.sre.shortName)-db-admin-username-$($databaseType.ToLower())"
             dbAdminPasswordSecretName = "$($config.sre.shortName)-db-admin-password-$($databaseType.ToLower())"
@@ -1092,7 +1095,8 @@ function Get-SreConfig {
                 }
             }
         }
-        if ($databaseType -eq "MSSQL") { $config.sre.databases["db$($databaseType.ToLower())"]["enableSSIS"] = $true }
+        $config.sre.databases.enabled = $True
+        if ($databaseType -eq "MSSQL") { $config.sre.databases.instances[-1].enableSSIS = $true }
         $ipOffset += 1
     }
 

diff --git a/deployment/common/DataStructures.psm1 b/deployment/common/DataStructures.psm1
@@ -151,6 +151,8 @@ function ConvertTo-SortedHashtable {
         $OutputHashtable = [ordered]@{}
         $Sortable.GetEnumerator() | Sort-Object -Property "Name" | ForEach-Object { $OutputHashtable.Add($_.Key, $(ConvertTo-SortedHashtable -Sortable $_.Value)) }
         return $OutputHashtable
+    } elseif ($Sortable -is [System.Object[]]) {
+        return @($Sortable | ForEach-Object { ConvertTo-SortedHashtable $_ })
     } else {
         return $Sortable
     }

diff --git a/deployment/secure_research_environment/network_rules/sre-nsg-rules-compute.json b/deployment/secure_research_environment/network_rules/sre-nsg-rules-compute.json
@@ -47,18 +47,24 @@
         "sourceAddressPrefix": "{{sre.network.vnet.subnets.compute.cidr}}",
         "sourcePortRange": "*"
     },
+    {{#sre.databases.enabled}}
     {
         "name": "AllowDatabasesSubnetOutbound",
         "access": "Allow",
         "description": "Allow outbound connections to the databases subnet",
         "destinationAddressPrefix": "{{sre.network.vnet.subnets.databases.cidr}}",
-        "destinationPortRange": ["{{sre.databases.dbmssql.port}}", "{{sre.databases.dbpostgresql.port}}"],
+        "destinationPortRange": [
+            {{#sre.databases.instances}}
+                "{{port}}",
+            {{/sre.databases.instances}}
+        ],
         "direction": "Outbound",
         "priority": 500,
         "protocol": "*",
         "sourceAddressPrefix": "{{sre.network.vnet.subnets.compute.cidr}}",
         "sourcePortRange": "*"
     },
+    {{/sre.databases.enabled}}
     {
         "name": "AllowWebappsSubnetOutbound",
         "access": "Allow",

diff --git a/deployment/secure_research_environment/network_rules/sre-nsg-rules-databases.json b/deployment/secure_research_environment/network_rules/sre-nsg-rules-databases.json
@@ -1,16 +1,22 @@
 [
+    {{#sre.databases.enabled}}
     {
         "name": "AllowComputeSubnetInbound",
         "access": "Allow",
         "description": "Allow inbound connections from the compute subnet",
         "destinationAddressPrefix": "{{sre.network.vnet.subnets.databases.cidr}}",
-        "destinationPortRange": ["{{sre.databases.dbmssql.port}}", "{{sre.databases.dbpostgresql.port}}"],
+        "destinationPortRange": [
+            {{#sre.databases.instances}}
+                "{{port}}",
+            {{/sre.databases.instances}}
+        ],
         "direction": "Inbound",
         "priority": 500,
         "protocol": "*",
         "sourceAddressPrefix": "{{sre.network.vnet.subnets.compute.cidr}}",
         "sourcePortRange": "*"
     },
+    {{/sre.databases.enabled}}
     {
         "name": "DenyAdminVPNInbound",
         "access": "Deny",

diff --git a/deployment/secure_research_environment/setup/Add_Single_SRD.ps1 b/deployment/secure_research_environment/setup/Add_Single_SRD.ps1
@@ -250,13 +250,21 @@ Copy-Item (Join-Path $PSScriptRoot ".." ".." "secure_research_desktop" "packages
 Copy-Item (Join-Path $PSScriptRoot ".." ".." ".." "tests" "srd_smoke_tests") -Filter *.* -Destination (Join-Path $localSmokeTestDir "tests") -Recurse
 # Expand mustache templates
 $PythonYaml = (ConvertFrom-Yaml (Get-Content -Raw (Join-Path $PSScriptRoot ".." ".." "secure_research_desktop" "packages" "packages-python.yaml")))
+$MssqlConfig = $config.sre.databases.instances | Where-Object { $_.type -eq "MSSQL" } | Select-Object -First 1
+$PostgresqlConfig = $config.sre.databases.instances | Where-Object { $_.type -eq "PostgreSQL" } | Select-Object -First 1
 $config["SmokeTests"] = [ordered]@{
-    PyPIPackage0 = Get-Content (Join-Path $PSScriptRoot ".." ".." ".." "environment_configs" "package_lists" "allowlist-full-python-pypi-tier3.list") -Head 1
-    PyPIPackage1 = Get-Content (Join-Path $PSScriptRoot ".." ".." ".." "environment_configs" "package_lists" "allowlist-full-python-pypi-tier3.list") -Tail 1
-    Python_v0    = $PythonYaml["versions"][0]
-    Python_v1    = $PythonYaml["versions"][1]
-    Python_v2    = $PythonYaml["versions"][2]
-    TestFailures = $config.sre.tier -ge 3 ? 1 : 0
+    MSSQLExists      = $MssqlConfig.Count -gt 0
+    MSSQLPort        = $MssqlConfig ? $MssqlConfig.port : ""
+    MSSQLVMName      = $MssqlConfig ? $MssqlConfig.vmName : ""
+    PostgreSQLExists = $PostgresqlConfig.Count -gt 0
+    PostgreSQLPort   = $PostgresqlConfig ? $PostgresqlConfig.port : ""
+    PostgreSQLVMName = $PostgresqlConfig ? $PostgresqlConfig.vmName : ""
+    PyPIPackageFirst = Get-Content (Join-Path $PSScriptRoot ".." ".." ".." "environment_configs" "package_lists" "allowlist-full-python-pypi-tier3.list") -Head 1
+    PyPIPackageLast  = Get-Content (Join-Path $PSScriptRoot ".." ".." ".." "environment_configs" "package_lists" "allowlist-full-python-pypi-tier3.list") -Tail 1
+    Python_v0        = $PythonYaml["versions"][0]
+    Python_v1        = $PythonYaml["versions"][1]
+    Python_v2        = $PythonYaml["versions"][2]
+    TestFailures     = $config.sre.tier -ge 3 ? 1 : 0
 }
 foreach ($MustacheFilePath in (Get-ChildItem -Path $localSmokeTestDir -Include *.mustache.* -File -Recurse)) {
     $ExpandedFilePath = $MustacheFilePath -replace ".mustache.", "."

diff --git a/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 b/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1
@@ -44,10 +44,7 @@ $deploymentSubnet = Get-Subnet -Name $config.sre.network.vnet.subnets.deployment
 
 # Create each database defined in the config file
 # -----------------------------------------------
-foreach ($keyName in $config.sre.databases.Keys) {
-    if ($config.sre.databases[$keyName] -isnot [System.Collections.IDictionary]) { continue }
-    $databaseCfg = $config.sre.databases[$keyName]
-
+foreach ($databaseCfg in $config.sre.databases.instances) {
     # Check whether this database VM has already been deployed
     # --------------------------------------------------------
     if (Get-AzVM -Name $databaseCfg.vmName -ResourceGroupName $config.sre.databases.rg -ErrorAction SilentlyContinue) {

diff --git a/docs/deployment/snippets/02_configuration.partial.md b/docs/deployment/snippets/02_configuration.partial.md
@@ -31,7 +31,7 @@ The following core SRE properties are required - look in the `environment_config
   "remoteDesktopProvider": "Which remote desktop provider to use. Either 'ApacheGuacamole' (recommended, tiers 0-3) or 'MicrosoftRDS' (tiers 2-3 only)",
   "azureAdminGroupName": "[Optional] Azure Security Group that admins of this SRE will belong to. If not specified then the same one as the SHM will be used.",
   "dataAdminIpAddresses": "A list of one or more IP addresses which admins will be using to transfer sensitive data to/from the secure Azure storage area (if not specified then Turing IP addresses will be used).",
-  "databases": "A list of one or more database flavours from the following list ('MSSQL', 'PostgreSQL'). For example ['MSSQL', 'PostgreSQL'] would deploy both an MS-SQL and a PostgreSQL database.",
+  "databases": [Optional] "A list of zero or more database flavours from the following list ('MSSQL', 'PostgreSQL'). For example ['MSSQL', 'PostgreSQL'] would deploy both an MS-SQL and a PostgreSQL database.",
   "deploymentIpAddresses": "[Optional] A list of one or more IP addresses which admins will be using when deploying the SRE (if not specified then deployment commands from any IP address will be permitted).",
   "domain": "[Optional] The fully qualified domain name for the SRE. If not specified then <SRE ID>.<SHM domain> will be used.",
   "overrides": "[Optional, Advanced] Do not use this unless you know what you're doing! If you want to override any of the default settings, you can do so by creating the same JSON structure that would be found in the final config file and nesting it under this entry. For example, to change the name of the Key Vault secret containing the MSSQL admin password, you could use something like: 'sre: { databases: { dbmssql: { adminPasswordSecretName: my-password-name } } }'"

diff --git a/tests/pester/ConfigurationFiles.Tests.ps1 b/tests/pester/ConfigurationFiles.Tests.ps1
@@ -19,7 +19,7 @@ Describe "SHM configuration file check" {
 
         # Load test config
         Mock Write-Information {} # we mock Write-Information here as we expect output from the `Get-SreConfig` call
-        $testConfig = Get-ShmConfig -shmId $ConfigId
+        $testConfig = Get-ShmConfig -shmId $ConfigId | ConvertTo-SortedHashtable
 
         # Compare the two configs as JSON strings
         # Note that we could use `Test-Equality` from the `Functional` module here, but that would not tell us *where* any differences are

diff --git a/tests/resources/sre_bluet1guac_full_config.json b/tests/resources/sre_bluet1guac_full_config.json
@@ -1107,51 +1107,54 @@
       }
     },
     "databases": {
-      "dbmssql": {
-        "adminPasswordSecretName": "sre-t1guac-vm-admin-password-mssql",
-        "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-mssql",
-        "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql",
-        "disks": {
-          "data": {
-            "sizeGb": "1024",
-            "type": "Standard_LRS"
+      "enabled": true,
+      "instances": [
+        {
+          "adminPasswordSecretName": "sre-t1guac-vm-admin-password-mssql",
+          "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql",
+          "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-mssql",
+          "vmName": "MSSQL-T1GUAC",
+          "type": "MSSQL",
+          "ip": "10.151.3.4",
+          "port": "1433",
+          "sku": "sqldev-gen2",
+          "subnet": "databases",
+          "vmSize": "Standard_DS2_v2",
+          "disks": {
+            "data": {
+              "sizeGb": "1024",
+              "type": "Standard_LRS"
+            },
+            "os": {
+              "sizeGb": "128",
+              "type": "Standard_LRS"
+            }
           },
-          "os": {
-            "sizeGb": "128",
-            "type": "Standard_LRS"
-          }
+          "enableSSIS": true
         },
-        "enableSSIS": true,
-        "ip": "10.151.3.4",
-        "port": "1433",
-        "sku": "sqldev-gen2",
-        "subnet": "databases",
-        "type": "MSSQL",
-        "vmName": "MSSQL-T1GUAC",
-        "vmSize": "Standard_DS2_v2"
-      },
-      "dbpostgresql": {
-        "adminPasswordSecretName": "sre-t1guac-vm-admin-password-postgresql",
-        "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-postgresql",
-        "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql",
-        "disks": {
-          "data": {
-            "sizeGb": "1024",
-            "type": "Standard_LRS"
-          },
-          "os": {
-            "sizeGb": "128",
-            "type": "Standard_LRS"
+        {
+          "adminPasswordSecretName": "sre-t1guac-vm-admin-password-postgresql",
+          "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql",
+          "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-postgresql",
+          "vmName": "PSTGRS-T1GUAC",
+          "type": "PostgreSQL",
+          "ip": "10.151.3.5",
+          "port": "5432",
+          "sku": "Ubuntu-latest",
+          "subnet": "databases",
+          "vmSize": "Standard_DS2_v2",
+          "disks": {
+            "data": {
+              "sizeGb": "1024",
+              "type": "Standard_LRS"
+            },
+            "os": {
+              "sizeGb": "128",
+              "type": "Standard_LRS"
+            }
           }
-        },
-        "ip": "10.151.3.5",
-        "port": "5432",
-        "sku": "Ubuntu-latest",
-        "subnet": "databases",
-        "type": "PostgreSQL",
-        "vmName": "PSTGRS-T1GUAC",
-        "vmSize": "Standard_DS2_v2"
-      },
+        }
+      ],
       "rg": "RG_SHM_BLUE_SRE_T1GUAC_DATABASES"
     },
     "diskTypeDefault": "Standard_LRS",

diff --git a/tests/resources/sre_bluet3msrds_full_config.json b/tests/resources/sre_bluet3msrds_full_config.json
@@ -1107,51 +1107,54 @@
       }
     },
     "databases": {
-      "dbmssql": {
-        "adminPasswordSecretName": "sre-t3msrds-vm-admin-password-mssql",
-        "dbAdminPasswordSecretName": "sre-t3msrds-db-admin-password-mssql",
-        "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-mssql",
-        "disks": {
-          "data": {
-            "sizeGb": "1024",
-            "type": "Standard_LRS"
+      "enabled": true,
+      "instances": [
+        {
+          "adminPasswordSecretName": "sre-t3msrds-vm-admin-password-mssql",
+          "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-mssql",
+          "dbAdminPasswordSecretName": "sre-t3msrds-db-admin-password-mssql",
+          "vmName": "MSSQL-T3MSRDS",
+          "type": "MSSQL",
+          "ip": "10.163.3.4",
+          "port": "1433",
+          "sku": "sqldev-gen2",
+          "subnet": "databases",
+          "vmSize": "Standard_DS2_v2",
+          "disks": {
+            "data": {
+              "sizeGb": "1024",
+              "type": "Standard_LRS"
+            },
+            "os": {
+              "sizeGb": "128",
+              "type": "Standard_LRS"
+            }
           },
-          "os": {
-            "sizeGb": "128",
-            "type": "Standard_LRS"
-          }
+          "enableSSIS": true
         },
-        "enableSSIS": true,
-        "ip": "10.163.3.4",
-        "port": "1433",
-        "sku": "sqldev-gen2",
-        "subnet": "databases",
-        "type": "MSSQL",
-        "vmName": "MSSQL-T3MSRDS",
-        "vmSize": "Standard_DS2_v2"
-      },
-      "dbpostgresql": {
-        "adminPasswordSecretName": "sre-t3msrds-vm-admin-password-postgresql",
-        "dbAdminPasswordSecretName": "sre-t3msrds-db-admin-password-postgresql",
-        "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-postgresql",
-        "disks": {
-          "data": {
-            "sizeGb": "1024",
-            "type": "Standard_LRS"
-          },
-          "os": {
-            "sizeGb": "128",
-            "type": "Standard_LRS"
+        {
+          "adminPasswordSecretName": "sre-t3msrds-vm-admin-password-postgresql",
+          "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-postgresql",
+          "dbAdminPasswordSecretName": "sre-t3msrds-db-admin-password-postgresql",
+          "vmName": "PSTGRS-T3MSRDS",
+          "type": "PostgreSQL",
+          "ip": "10.163.3.5",
+          "port": "5432",
+          "sku": "Ubuntu-latest",
+          "subnet": "databases",
+          "vmSize": "Standard_DS2_v2",
+          "disks": {
+            "data": {
+              "sizeGb": "1024",
+              "type": "Standard_LRS"
+            },
+            "os": {
+              "sizeGb": "128",
+              "type": "Standard_LRS"
+            }
           }
-        },
-        "ip": "10.163.3.5",
-        "port": "5432",
-        "sku": "Ubuntu-latest",
-        "subnet": "databases",
-        "type": "PostgreSQL",
-        "vmName": "PSTGRS-T3MSRDS",
-        "vmSize": "Standard_DS2_v2"
-      },
+        }
+      ],
       "rg": "RG_SHM_BLUE_SRE_T3MSRDS_DATABASES"
     },
     "diskTypeDefault": "Standard_LRS",