Merge pull request #1955 from alan-turing-institute/release-v4.2.2

Release v4.2.2
alan-turing-institute · Jul 15, 2024 · 008d346 · 008d346
2 parents bee9fc4 + 1259288
commit 008d346
Show file tree

Hide file tree

Showing 27 changed files with 122 additions and 118 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -11,12 +11,12 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
     lsb-release \
     python3-sphinx \
     sudo
-    
+
 # Set package versions
 ARG AZURE_CLI_VERSION="2.58.0"
-ARG PWSH_VERSION="7.4.1"
+ARG PWSH_VERSION="7.4.3"
 
-# Install Azure-CLI 
+# Install Azure-CLI
 # Get Microsoft signing key
 RUN sudo mkdir -p /etc/apt/keyrings \
   && curl -sLS https://packages.microsoft.com/keys/microsoft.asc | \

diff --git a/.lychee.toml b/.lychee.toml
@@ -95,6 +95,7 @@ exclude = [
   'ipaddressguide\.com',  # 403
   'opensource\.org',  # 403
   'portal\.azure\.com',  # 403
+  'web\.archive\.org', # DDOS protection
 ]
 
 # Exclude these filesystem paths from getting checked.

diff --git a/SECURITY.md b/SECURITY.md
@@ -7,8 +7,8 @@ All organisations using an earlier version in production should update to the la
 
 | Version                                                                                 | Supported          |
 | --------------------------------------------------------------------------------------- | ------------------ |
-| [4.2.1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.2.1)   | :white_check_mark: |
-| < 4.2.1                                                                                 | :x:                |
+| [4.2.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.2.2)   | :white_check_mark: |
+| < 4.2.2                                                                                 | :x:                |
 
 ## Reporting a Vulnerability
 

diff --git a/deployment/CheckRequirements.ps1 b/deployment/CheckRequirements.ps1
@@ -13,7 +13,7 @@ $ModuleVersionRequired = @{
     "Az.Accounts"                                  = @("ge", "2.11.1")
     "Az.Automation"                                = @("ge", "1.9.0")
     "Az.Compute"                                   = @("ge", "5.3.0")
-    "Az.DataProtection"                            = @("ge", "0.4.0")
+    "Az.DataProtection"                            = @("ge", "2.4.0")
     "Az.Dns"                                       = @("ge", "1.1.2")
     "Az.KeyVault"                                  = @("ge", "4.9.1")
     "Az.Monitor"                                   = @("ge", "4.2.0")
@@ -28,6 +28,7 @@ $ModuleVersionRequired = @{
     "Microsoft.Graph.Applications"                 = @("ge", "1.21.0")
     "Microsoft.Graph.Identity.DirectoryManagement" = @("ge", "1.21.0")
     "Microsoft.Graph.Users"                        = @("ge", "1.21.0")
+    "Posh-ACME"                                    = @("ge", "4.23.0")
     "Poshstache"                                   = @("ge", "0.1.10")
     "Powershell-Yaml"                              = @("ge", "0.4.2")
 }

diff --git a/deployment/common/AzureDataProtection.psm1 b/deployment/common/AzureDataProtection.psm1
@@ -157,11 +157,12 @@ function Deploy-DataProtectionBackupPolicy {
             Edit-AzDataProtectionPolicyTriggerClientObject -Policy $Template `
                                                            -Schedule $Schedule
         } elseif ($DataSourcetype -eq 'blob') {
-            # Modify default for retention to 12 weeks
-            # Modifying the object directly seems required as,
-            # Adding New Retention Rule is not supported for AzureBlob datasource Type.
-            # Removing Default Retention Rule is not allowed. Please try again with different rule name.
-            $Template.PolicyRule.Lifecycle.DeleteAfterDuration = "P12W"
+            Edit-AzDataProtectionPolicyTriggerClientObject -Policy $Template -RemoveSchedule
+            # Change default retention period to 12 weeks
+            $lifeCycleOperationalTier = New-AzDataProtectionRetentionLifeCycleClientObject -SourceDataStore OperationalStore `
+                                                                                           -SourceRetentionDurationType Weeks `
+                                                                                           -SourceRetentionDurationCount 12
+            Edit-AzDataProtectionPolicyRetentionRuleClientObject -Policy $Template -Name Default -LifeCycles $lifeCycleOperationalTier -IsDefault $true -OverwriteLifeCycle $true
         }
         $Policy = New-AzDataProtectionBackupPolicy -ResourceGroupName $ResourceGroupName `
                                                    -VaultName $VaultName `

diff --git a/deployment/common/AzureNetwork.psm1 b/deployment/common/AzureNetwork.psm1
@@ -408,9 +408,6 @@ function Deploy-PublicIpAddress {
     if ($notExists) {
         Add-LogMessage -Level Info "[ ] Creating public IP address '$Name'"
         $ipAddressParams = @{}
-        if ($Sku -eq "Standard") {
-            $ipAddressParams["Zone"] = @(1, 2, 3)
-        }
         $publicIpAddress = New-AzPublicIpAddress -Name $Name -ResourceGroupName $ResourceGroupName -AllocationMethod $AllocationMethod -Location $Location -Sku $Sku @ipAddressParams
         if ($?) {
             Add-LogMessage -Level Success "Created public IP address '$Name'"

diff --git a/deployment/common/Configuration.psm1 b/deployment/common/Configuration.psm1
@@ -83,7 +83,9 @@ function Get-ShmConfig {
         vmImagesRgPrefix    = $shmConfigBase.vmImages.rgPrefix ? $shmConfigBase.vmImages.rgPrefix : "RG_VMIMAGES"
         storageTypeDefault  = "Standard_GRS"
         diskTypeDefault     = "Standard_LRS"
-}
+        dockerAccount       = $shmConfigBase.docker.account ? $shmConfigBase.docker.account : "NA"
+        dockerPassword      = $shmConfigBase.docker.password ? $shmConfigBase.docker.password : "NA"
+    }
     # For normal usage this does not need to be user-configurable.
     # However, if you are migrating an existing SHM you will need to ensure that the address spaces of the SHMs do not overlap
     $shmIpPrefix = $shmConfigBase.overrides.ipPrefix ? $shmConfigBase.overrides.ipPrefix : "10.0.0"
@@ -327,7 +329,7 @@ function Get-ShmConfig {
             externalIpAddresses = [ordered]@{
                 linux   = (
                     @("72.32.157.246", "87.238.57.227", "147.75.85.69", "217.196.149.55") + # apt.postgresql.org
-                    @("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
+                    @("91.189.88.0/21", "185.125.188.0/22") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
                     $cloudFlareIpAddresses + # database.clamav.net, packages.gitlab.com and qgis.org use Cloudflare
                     $cloudFrontIpAddresses + # packages.gitlab.com uses Cloudfront to host its Release file
                     @("104.131.190.124") + # dbeaver.io
@@ -431,7 +433,7 @@ function Get-ShmConfig {
         hostname                   = $hostname
         hostnameLower              = $hostname.ToLower()
         hostnameUpper              = $hostname.ToUpper()
-        fqdn                       = "${hostname}.$($shm.domain.fqdn)"
+        fqdn                       = "${hostname}.$($shm.domain.fqdn)".ToLower()
         ip                         = Get-NextAvailableIpInRange -IpRangeCidr $shm.network.vnet.subnets.identity.cidr -Offset 4
         external_dns_resolver      = "168.63.129.16"  # https://docs.microsoft.com/en-us/azure/virtual-network/what-is-ip-address-168-63-129-16
         installationDirectory      = "C:\Installation"
@@ -451,7 +453,7 @@ function Get-ShmConfig {
     $shm.dcb = [ordered]@{
         vmName   = $hostname
         hostname = $hostname
-        fqdn     = "${hostname}.$($shm.domain.fqdn)"
+        fqdn     = "${hostname}.$($shm.domain.fqdn)".ToLower()
         ip       = Get-NextAvailableIpInRange -IpRangeCidr $shm.network.vnet.subnets.identity.cidr -Offset 5
     }
 
@@ -613,10 +615,12 @@ function Get-SreConfig {
     $sreConfigBase = Get-CoreConfig -shmId $shmId -sreId $sreId
 
     # Support for "MicrosoftRDS" has been removed. The "remotedDesktopProvider" field now defaults to "ApacheGuacamole"
-    if ($sreConfigBase.remoteDesktopProvider -ne "ApacheGuacamole") {
-        Add-LogMessage -Level Fatal "Support for remote desktops other than ApacheGuacamole has been removed"
-    } elseif ($sreConfigBase.remoteDesktopProvider -eq "ApacheGuacamole") {
-        Add-LogMessage -Level Warning "The remoteDesktopProvider configuration option has been deprecated and will be removed in the future"
+    if ($null -ne $sreConfigBase.remoteDesktopProvider) {
+        if ($sreConfigBase.remoteDesktopProvider -eq "ApacheGuacamole") {
+            Add-LogMessage -Level Warning "The remoteDesktopProvider configuration option has been deprecated and will be removed in the future"
+        } else {
+            Add-LogMessage -Level Fatal "Support for remote desktops other than ApacheGuacamole has been removed"
+        }
     }
     $sreConfigBase.remoteDesktopProvider = "ApacheGuacamole"
 
@@ -661,7 +665,7 @@ function Get-SreConfig {
     $sreDomain = $sreConfigBase.domain ? $sreConfigBase.domain : "$($config.sre.id).$($config.shm.domain.fqdn)".ToLower()
     $config.sre.domain = [ordered]@{
         dn          = "DC=$($sreDomain.Replace('.',',DC='))"
-        fqdn        = $sreDomain
+        fqdn        = "$sreDomain".ToLower()
         netbiosName = $($config.sre.id).ToUpper() | Limit-StringLength -MaximumLength 15 -FailureIsFatal
     }
     $config.sre.domain.securityGroups = [ordered]@{
@@ -892,7 +896,7 @@ function Get-SreConfig {
     foreach ($server in $config.sre.remoteDesktop.Keys) {
         if (-not $config.sre.remoteDesktop[$server].vmName) { continue }
         $config.sre.remoteDesktop[$server].hostname = $config.sre.remoteDesktop[$server].vmName
-        $config.sre.remoteDesktop[$server].fqdn = "$($config.sre.remoteDesktop[$server].vmName).$($config.shm.domain.fqdn)"
+        $config.sre.remoteDesktop[$server].fqdn = "$($config.sre.remoteDesktop[$server].vmName).$($config.shm.domain.fqdn)".ToLower()
     }
 
     # Set the appropriate tier-dependent network rules for the remote desktop server
@@ -980,7 +984,7 @@ function Get-SreConfig {
     # Construct the hostname and FQDN for each VM
     foreach ($server in $config.sre.webapps.Keys) {
         if ($config.sre.webapps[$server] -IsNot [System.Collections.Specialized.OrderedDictionary]) { continue }
-        $config.sre.webapps[$server].fqdn = "$($config.sre.webapps[$server].hostname).$($config.sre.domain.fqdn)"
+        $config.sre.webapps[$server].fqdn = "$($config.sre.webapps[$server].hostname).$($config.sre.domain.fqdn)".ToLower()
         $config.sre.webapps[$server].vmName = "$($config.sre.webapps[$server].hostname)-SRE-$($config.sre.id)".ToUpper()
     }
 

diff --git a/...nt/safe_haven_management_environment/cloud_init/cloud-init-repository-proxy.mustache.yaml b/...nt/safe_haven_management_environment/cloud_init/cloud-init-repository-proxy.mustache.yaml
@@ -68,6 +68,10 @@ write_files:
     content: |
       /usr/local/bin/configure-nexus --admin-password {{perInstance.nexusAdminPassword}} update-allowlists --tier {{perInstance.tier}} --pypi-package-file /etc/nexus/allowlist-pypi --cran-package-file /etc/nexus/allowlist-cran >> /var/log/configure_nexus.log 2>&1
 
+  - path: "/opt/configuration/docker_pat.txt"
+    permissions: "0400"
+    content: {{dockerPassword}}
+
 # Set locale and timezone
 locale: en_GB.UTF-8
 timezone: {{time.timezone.linux}}
@@ -146,7 +150,7 @@ runcmd:
   - sleep 1m
   - systemctl status docker
   - docker --version
-  - docker compose --version
+  - docker compose version
 
   # Create directory for Nexus data that is owned by the correct user inside the Docker container
   - echo ">=== Creating Nexus data directory... ===<"
@@ -163,7 +167,9 @@ runcmd:
 
   # Set up the Nexus container
   - echo ">=== Creating Nexus container... ===<"
-  - su nexusdaemon -c "docker compose -f /etc/nexus/docker-compose.yaml up -d"
+  - chown nexusdaemon:nexusdaemon /opt/configuration/docker_pat.txt  # Ensure that the file is owned by the nexusdaemon user
+  - su nexusdaemon -c "cat /opt/configuration/docker_pat.txt | docker login --username '{{dockerAccount}}' --password-stdin
+    && docker compose -f /etc/nexus/docker-compose.yaml up -d"
 
   # Give Nexus some time to initialise
   - echo ">=== Waiting for Nexus to initialise (5 minutes)... ===<"

diff --git a/deployment/safe_haven_management_environment/setup/Setup_SHM_Networking.ps1 b/deployment/safe_haven_management_environment/setup/Setup_SHM_Networking.ps1
@@ -102,7 +102,7 @@ $deploymentTier3Subnet = Set-SubnetNetworkSecurityGroup -Subnet $deploymentTier3
 
 # Create the VPN gateway
 # ----------------------
-$publicIp = Deploy-PublicIpAddress -Name "$($config.network.vnet.name)_GW_PIP" -ResourceGroupName $config.network.vnet.rg -AllocationMethod Dynamic -Location $config.location
+$publicIp = Deploy-PublicIpAddress -Name "$($config.network.vnet.name)_GW_PIP" -ResourceGroupName $config.network.vnet.rg -AllocationMethod Static -Location $config.location -Sku Standard
 $certificate = Resolve-KeyVaultSecret -VaultName $config.keyVault.name -SecretName $config.keyVault.secretNames.vpnCaCertificatePlain -AsPlaintext
 $null = Deploy-VirtualNetworkGateway -Name "$($config.network.vnet.name)_GW" -ResourceGroupName $config.network.vnet.rg -Location $config.location -PublicIpAddressId $publicIp.Id -SubnetId $gatewaySubnet.Id -P2SCertificate $certificate -VpnClientAddressPool $config.network.vpn.cidr
 

diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml
@@ -89,6 +89,10 @@ write_files:
     content: |
       {{set_dns.mustache.sh}}
 
+  - path: "/opt/configuration/docker_pat.txt"
+    permissions: "0400"
+    content: {{shm.dockerPassword}}
+
 # Set locale and timezone
 locale: en_GB.UTF-8
 timezone: {{sre.time.timezone.linux}}
@@ -204,7 +208,7 @@ runcmd:
   - sleep 1m
   - systemctl status docker
   - docker --version
-  - docker compose --version
+  - docker compose version
 
   # Set up the codimddaemon user
   - echo ">=== Configuring codimddaemon user... ===<"
@@ -216,7 +220,9 @@ runcmd:
 
   # Deploy CodiMD using Docker
   - echo ">=== Deploying CodiMD with Docker... ===<"
-  - su codimddaemon -c "docker compose -f /opt/codimd/docker-compose.yml up -d"
+  - chown codimddaemon:codimddaemon /opt/configuration/docker_pat.txt  # Ensure that the file is owned by the codimddaemon user
+  - su codimddaemon -c "cat /opt/configuration/docker_pat.txt | docker login --username '{{shm.dockerAccount}}' --password-stdin
+    && docker compose -f /opt/codimd/docker-compose.yml up -d"
 
   # Wait for deployment to finish
   - |

diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-guacamole.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-guacamole.mustache.yaml
@@ -84,6 +84,10 @@ write_files:
     content: |
       SRD Main;{{guacamole.ipAddressFirstSRD}}
 
+  - path: "/opt/configuration/docker_pat.txt"
+    permissions: "0400"
+    content: {{shm.dockerPassword}}
+
 # Set locale and timezone
 locale: en_GB.UTF-8
 timezone: {{sre.time.timezone.linux}}
@@ -170,7 +174,7 @@ runcmd:
   - sleep 1m
   - systemctl status docker
   - docker --version
-  - docker compose --version
+  - docker compose version
 
   # Set up the guacamoledaemon user
   - echo ">=== Configuring guacamoledaemon user... ===<"
@@ -197,6 +201,8 @@ runcmd:
 
   # Deploy Guacamole using Docker
   - echo ">=== Deploying Guacamole with Docker...  ===<"
+  - chown guacamoledaemon:guacamoledaemon /opt/configuration/docker_pat.txt  # Ensure that the file is owned by the codimddaemon user
+  - su guacamoledaemon -c "cat /opt/configuration/docker_pat.txt | docker login --username {{shm.dockerAccount}} --password-stdin"
   - su guacamoledaemon -c "docker compose -f /opt/guacamole/docker-compose.yaml up -d"
 
   # Generate the necessary SQL config for the local PostgreSQL database and run it

diff --git a/deployment/secure_research_environment/setup/Update_SRE_SSL_Certificate.ps1 b/deployment/secure_research_environment/setup/Update_SRE_SSL_Certificate.ps1
@@ -16,8 +16,8 @@ Import-Module Az.Compute
 Import-Module Az.KeyVault
 Import-Module $PSScriptRoot/../../common/AzureCompute -Force -ErrorAction Stop
 Import-Module $PSScriptRoot/../../common/AzureKeyVault -Force -ErrorAction Stop
-Import-Module $PSScriptRoot/../../common/Configuration -ErrorAction Stop
-Import-Module $PSScriptRoot/../../common/Logging -ErrorAction Stop
+Import-Module $PSScriptRoot/../../common/Configuration -Force -ErrorAction Stop
+Import-Module $PSScriptRoot/../../common/Logging -Force -ErrorAction Stop
 
 
 # Check that we are authenticated in Azure

diff --git a/docs/source/deployment/deploy_shm.md b/docs/source/deployment/deploy_shm.md
@@ -52,6 +52,12 @@ Alternatively, you may run multiple SHMs concurrently, for example you may have
 
     - ![Linux](https://img.shields.io/badge/-555?&logo=linux&logoColor=white) use your favourite package manager or install manually following the [instructions on GitHub](https://github.com/openssl/openssl)
 
+- `Docker Hub` account
+    - The DSH makes use of several public Docker images.
+      Due to Docker Hub download [rate limits](https://docs.docker.com/docker-hub/download-rate-limit/), we now require Docker credentials to ensure that all images are successfully downloaded at the time of deployment.
+    - We recommend using a personal access token (PAT) with Public Repo Read-Only permissions rather than your Docker account password.
+      See [instructions on Docker](https://docs.docker.com/security/for-developers/access-tokens/) for details of how to create a PAT.
+
 ````{hint}
 If you run:
 
@@ -118,6 +124,10 @@ The following core SHM properties are required - look in the `environment_config
     "location": "[Optional] Azure location where VM images should be built (if not specified then the value from the 'azure' block will be used). Multiple Safe Haven deployments can share a single set of VM images in a common subscription if desired - this is what is done in the Turing deployment. If you are hoping to use images that have already been built for another Safe Haven deployment, make sure you specify this parameter accordingly.",
     "buildIpAddresses": "[Optional] One or more IP addresses which admins will be running the VM build scripts from (if not specified then Turing IP addresses will be used)."
   },
+  "docker": {
+    "account": "A Docker Hub account name.",
+    "password": "The password or personal access token for the above account. We strongly recommend using a Personal Access Token with permissions set to Public Repo Read-only"
+  },
   "overrides": "[Optional, Advanced] Do not use this unless you know what you're doing! If you want to override any of the default settings, you can do so by creating the same JSON structure that would be found in the final config file and nesting it under this entry. For example, to change the size of the data disk on the domain controller, you could use something like: 'shm: { dc: { disks: { data: { sizeGb: 50 } } } }'"
 }
 ```
@@ -603,7 +613,7 @@ Note that a full set of {ref}`policy_tier_2` local mirrors currently take around
   </details>
 
 - Unzip the zip file and identify the root certificate (`Generic\VpnServerRoot.cer`) and VPN configuration file (`Generic\VpnSettings.xml`)
-- Follow the [VPN set up instructions](https://docs.microsoft.com/en-us/azure/vpn-gateway/point-to-site-vpn-client-configuration-azure-cert) using the section appropriate to your operating system (**you do not need to install the `Generic\VpnServerRoot.cer` certificate, as we're using our own self-signed root certificate**):
+- Follow the [VPN set up instructions](https://web.archive.org/web/20240527120727/https://learn.microsoft.com/en-us/azure/vpn-gateway/point-to-site-vpn-client-cert-windows) using the section appropriate to your operating system (**you do not need to install the `Generic\VpnServerRoot.cer` certificate, as we're using our own self-signed root certificate**):
 
 ```{admonition} ![Windows](https://img.shields.io/badge/-555?&logo=windows&logoColor=white) instructions
 - Use SSTP for the VPN type