alan-turing-institute · JimMadge · Oct 24, 2022 · Jan 5, 2023 · Jan 27, 2023 · Jul 26, 2023
@@ -6,7 +6,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 
 # Set package versions
 ARG AZURE_CLI_VERSION="2.42.0"
-ARG PWSH_VERSION="7.3.2"
+ARG PWSH_VERSION="7.3.6"
 
 # Set up TARGETARCH variable to use to pull the right binaries for the current architecture.
 ARG TARGETARCH

@@ -64,6 +64,7 @@ We usually deploy the latest available version of the Data Safe Haven for each o
 | December 2021    | DSG 2021-12                   | [v3.3.1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.3.1)           |
 | December 2022    | DSG 2022-12                   | [v4.0.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.2)           |
 | February 2023    | DSG 2023-02                   | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3)           |
+| May 2023         | DSG 2023-05                   | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3)           |
 
 ## Versions that have undergone formal security evaluation
 

@@ -8,7 +8,7 @@ param (
 Import-Module $PSScriptRoot/common/Logging -Force -ErrorAction Stop
 
 # Requirements
-$PowershellSupportedVersion = "7.3.2"
+$PowershellSupportedVersion = "7.3.6"
 $ModuleVersionRequired = @{
     "Az.Accounts"                                  = @("ge", "2.11.1")
     "Az.Automation"                                = @("ge", "1.9.0")

@@ -332,7 +332,7 @@ function Get-ShmConfig {
                 ) # *-jobruntimedata-prod-su1.azure-automation.net
                 linux           = (
                     @("72.32.157.246", "87.238.57.227", "147.75.85.69", "217.196.149.55") + # apt.postgresql.org
-                    @("91.189.91.38", "91.189.91.39", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
+                    @("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
                     $cloudFlareIpAddresses + # database.clamav.net, packages.gitlab.com and qgis.org use Cloudflare
                     $cloudFrontIpAddresses + # packages.gitlab.com uses Cloudfront to host its Release file
                     @("104.131.190.124") + # dbeaver.io

@@ -9,7 +9,7 @@ disk_setup:
 
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 
 mounts:

@@ -9,7 +9,7 @@ disk_setup:
 
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 
 mounts:
@@ -122,7 +122,7 @@ write_files:
   - path: "/etc/cron.d/pull-from-internet"
     permissions: "0644"
     content: |
-      # External update (rsync from CRAN) every 6 hours
+      # External update from PyPi every 6 hours
       0 */6 * * * mirrordaemon ~mirrordaemon/pull_from_internet.sh
 
   - path: "/etc/cron.d/pull-then-push"
@@ -313,7 +313,7 @@ runcmd:
 
   # Install bandersnatch with pip
   - echo ">=== Installing bandersnatch... ===<"
-  - pip3 install bandersnatch==4.2.0
+  - pip3 install bandersnatch==4.2.0 packaging==21.3
   - echo "Using bandersnatch from '$(which bandersnatch)'"
 
   # Initialise allowlist if appropriate

@@ -9,7 +9,7 @@ disk_setup:
 
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 
 mounts:

@@ -9,7 +9,7 @@ disk_setup:
 
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 
 mounts:

@@ -114,11 +114,6 @@ write_files:
     content: |
       {{packages-python.yaml}}
 
-  - path: "/opt/build/pyenv/pyproject_template.toml"
-    permissions: "0400"
-    content: |
-      {{pyenv_pyproject_template.toml}}
-
   - path: "/opt/build/rstudio.debinfo"
     permissions: "0400"
     content: |

@@ -114,11 +114,6 @@ write_files:
     content: |
       {{packages-python.yaml}}
 
-  - path: "/opt/build/pyenv/pyproject_template.toml"
-    permissions: "0400"
-    content: |
-      {{pyenv_pyproject_template.toml}}
-
   - path: "/opt/build/rstudio.debinfo"
     permissions: "0400"
     content: |

@@ -118,11 +118,6 @@ write_files:
     content: |
       {{packages-python.yaml}}
 
-  - path: "/opt/build/pyenv/pyproject_template.toml"
-    permissions: "0400"
-    content: |
-      {{pyenv_pyproject_template.toml}}
-
   - path: "/opt/build/rbase.debinfo"
     permissions: "0400"
     content: |

@@ -9,7 +9,6 @@ if [ $# -ne 1 ]; then
 fi
 PYTHON_VERSION=$1
 PYENV_ROOT="$(pyenv root)"
-PYPROJECT_TOML="/opt/build/python-${PYTHON_VERSION}-pyproject.toml"
 MONITORING_LOG="/opt/monitoring/python-${PYTHON_VERSION}-package-versions.log"
 REQUIREMENTS_TXT="/opt/build/python-${PYTHON_VERSION}-requirements.txt"
 REQUESTED_PACKAGE_LIST="/opt/build/packages/packages-python-${PYTHON_VERSION}.list"
@@ -27,24 +26,19 @@ echo "Installed $(${EXE_PATH}/python --version)"
 # Install and upgrade installation prerequisites
 # ----------------------------------------------
 echo "Installing and upgrading installation prerequisites for Python ${PYTHON_VERSION}..."
-${EXE_PATH}/pip install --upgrade pip poetry
+${EXE_PATH}/pip install --upgrade pip pip-tools setuptools
 
 
-# Solve dependencies and install using poetry
-# -------------------------------------------
-echo "Installing packages with poetry..."
-${EXE_PATH}/poetry config virtualenvs.create false
-${EXE_PATH}/poetry config virtualenvs.in-project true
-rm poetry.lock pyproject.toml 2> /dev/null
-sed -e "s/PYTHON_VERSION/$PYTHON_VERSION/" /opt/build/pyenv/pyproject_template.toml > $PYPROJECT_TOML
-ln -s $PYPROJECT_TOML pyproject.toml
-${EXE_PATH}/poetry add $(tr '\n' ' ' < $REQUIREMENTS_TXT) || exit 3
+# Solve dependencies and write package versions to monitoring log
+# ---------------------------------------------------------------
+echo "Determining package versions with pip-compile..."
+${EXE_PATH}/pip-compile -o "$MONITORING_LOG" "$REQUIREMENTS_TXT"
 
 
-# Write package versions to monitoring log
-# ----------------------------------------
-${EXE_PATH}/poetry show > $MONITORING_LOG
-${EXE_PATH}/poetry show --tree >> $MONITORING_LOG
+# Install pinned packages using pip
+# ---------------------------------
+echo "Installing packages with pip..."
+${EXE_PATH}/pip install -r "$MONITORING_LOG"
 
 
 # Run any post-install commands

@@ -39,9 +39,8 @@ packages:
     pathos:
     pg8000:
     Pillow:
+    pip-tools:
     plotly:
-    poetry:  # also used by installation scripts
-        "all": [">1.0.0"]  # increase solver flexibility
     prophet:
     psycopg2:
     pydot:

@@ -1,3 +1,4 @@
+arrow
 BiocManager
 caret
 csv

@@ -161,7 +161,7 @@ disk_setup:
     overwrite: true
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 mounts:
   - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"]

@@ -132,7 +132,7 @@ disk_setup:
     overwrite: true
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 mounts:
   - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"]

@@ -122,7 +122,7 @@ disk_setup:
     overwrite: true
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 mounts:
   - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"]

@@ -25,7 +25,7 @@ disk_setup:
 
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
-    partition: 1
+    partition: auto
     filesystem: ext4
 
 mounts:

@@ -12,7 +12,7 @@ disk_setup:
 fs_setup:
   - device: /dev/disk/azure/scsi1/lun1
     filesystem: ext4
-    partition: 1
+    partition: auto
 
 # Note that we do not include the blobfuse mounts here as these are controlled by systemd
 mounts:

@@ -107,7 +107,7 @@ if ($operationFailed -Or (-Not $loginExists)) {
         # Create a DB user for each login group
         Write-Output "Ensuring that an SQL user exists for '$domainGroup' on: '$serverName'..."
         $sqlCommand = "IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$domainGroup') CREATE USER [$domainGroup] FOR LOGIN [$domainGroup];"
-        Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
+        Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query "$sqlCommand" -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
         if ($? -And -Not $sqlErrorMessage) {
             Write-Output " [o] Ensured that '$domainGroup' user exists on: '$serverName'"
             Start-Sleep -s 10  # allow time for the database action to complete
@@ -124,7 +124,7 @@ if ($operationFailed -Or (-Not $loginExists)) {
     foreach ($groupSchemaTuple in @(($DataAdminGroup, "data"), ($ResearchUsersGroup, "dbopublic"))) {
         $domainGroup, $schemaName = $groupSchemaTuple
         $sqlCommand = "IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = N'$schemaName') EXEC('CREATE SCHEMA $schemaName AUTHORIZATION [$domainGroup]');"
-        Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
+        Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
         if ($? -And -Not $sqlErrorMessage) {
             Write-Output " [o] Successfully ensured that '$schemaName' schema exists on: '$serverName'"
             Start-Sleep -s 10  # allow time for the database action to complete
@@ -154,7 +154,7 @@ if ($operationFailed -Or (-Not $loginExists)) {
             Write-Output " [x] Role $role not recognised!"
             continue
         }
-        Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
+        Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
         if ($? -And -Not $sqlErrorMessage) {
             Write-Output " [o] Successfully gave '$domainGroup' $role permissions on: '$serverName'"
             Start-Sleep -s 10  # allow time for the database action to complete
@@ -171,7 +171,7 @@ if ($operationFailed -Or (-Not $loginExists)) {
     # ------------------------------------
     Write-Output "Running T-SQL lockdown script on: '$serverName'..."
     $sqlCommand = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ServerLockdownCommandB64))
-    Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
+    Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
     if ($? -And -Not $sqlErrorMessage) {
         Write-Output " [o] Successfully ran T-SQL lockdown script on: '$serverName'"
     } else {
@@ -187,7 +187,7 @@ if ($operationFailed -Or (-Not $loginExists)) {
     $windowsAdmin = "${serverName}\${VmAdminUsername}"
     Write-Output "Removing database access from $windowsAdmin on: '$serverName'..."
     $sqlCommand = "DROP USER IF EXISTS [$windowsAdmin]; IF EXISTS(SELECT * FROM master.dbo.syslogins WHERE loginname = '$windowsAdmin') DROP LOGIN [$windowsAdmin]"
-    Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
+    Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
     if ($? -And -Not $sqlErrorMessage) {
         Write-Output " [o] Successfully removed database access for $windowsAdmin on: '$serverName'"
         Start-Sleep -s 10  # allow time for the database action to complete
@@ -203,7 +203,7 @@ if ($operationFailed -Or (-Not $loginExists)) {
     # ---------------------------------------------------------------------------------
     Write-Output "Revoking sysadmin role from $DbAdminUsername on: '$serverName'..."
     $sqlCommand = "ALTER SERVER ROLE sysadmin DROP MEMBER $DbAdminUsername;"
-    Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
+    Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true
     if ($? -And -Not $sqlErrorMessage) {
         Write-Output " [o] Successfully revoked sysadmin role on: '$serverName'"
         Start-Sleep -s 10  # allow time for the database action to complete

@@ -26,7 +26,8 @@ $null = Set-AzContext -SubscriptionId $config.sre.subscriptionName -ErrorAction
 # --------------------------------------
 $firewallRules = Get-JsonFromMustacheTemplate -TemplatePath (Join-Path $PSScriptRoot ".." ".." "safe_haven_management_environment" "network_rules" "shm-firewall-rules.json") -Parameters $config.shm -AsHashtable
 $allowedFqdns = @($firewallRules.applicationRuleCollections | ForEach-Object { $_.properties.rules.targetFqdns }) +
-                @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name })
+                @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name }) +
+                @("docker.io")
 # List all unique FQDNs
 $allowedFqdns = $allowedFqdns |
                 Where-Object { $_ -notlike "*-sb.servicebus.windows.net" } | # Remove AzureADConnect password reset endpoints

@@ -58,7 +58,6 @@ foreach ($databaseCfg in $config.sre.databases.instances) {
             }
         } else {
             Add-LogMessage -Level Warning "Database VM '$($databaseCfg.vmName)' already exists. Use the '-Redeploy' option if you want to remove the existing database and its data and deploy a new one."
-            continue
         }
     }
 

@@ -108,7 +108,7 @@ PS> ./Provision_Compute_VM.ps1 -shmId <SHM ID>
 
 ```{note}
 - Although the `./Provision_Compute_VM.ps1` script will finish running in a few minutes, the build itself will take several hours.
-- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/verification/analyse_build.py`.
+- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/monitoring/analyse_build.py`.
 - **NB.** You will need to connect from an approved administrator IP address
 - **NB.** the VM will automatically shutdown at the end of the cloud-init process - if you want to analyse the build after this point, you will need to turn it back on in the `Azure` portal.
 ```

@@ -105,6 +105,42 @@ PS> ./Setup_SRE_Guacamole_Servers.ps1 -shmId <SHM ID> -sreId <SRE ID>
 
 </details>
 
+<details>
+<summary><strong>Update SSL certificate</strong></summary>
+
+![Powershell: five minutes](https://img.shields.io/static/v1?style=for-the-badge&logo=powershell&label=local&color=blue&message=five%20minutes) at {{file_folder}} `./deployment/secure_research_environment/setup`
+
+```powershell
+PS> ./Update_SRE_SSL_Certificate.ps1 -shmId <SHM ID> -sreId <SRE ID>
+```
+
+- where `<SHM ID>` is the {ref}`management environment ID <roles_deployer_shm_id>` for this SHM
+- where `<SRE ID>` is the {ref}`secure research environment ID <roles_deployer_sre_id>` for this SRE
+- where `<email>` is an email address that you want to be notified when certificates are close to expiry
+
+```{tip}
+`./Update_SRE_RDS_SSL_Certificate.ps1` should be run again whenever you want to update the certificate for this SRE.
+```
+
+```{caution}
+`Let's Encrypt` will only issue **5 certificates per week** for a particular host (e.g. `rdg-sre-sandbox.project.turingsafehaven.ac.uk`).
+To reduce the number of calls to `Let's Encrypt`, the signed certificates are stored in the Key Vault for easy redeployment.
+For production environments this should usually not be an issue.
+```
+
+````{important}
+If you find yourself frequently redeploying a test environment and hit the `Let's Encrypt` certificate limit, you can can use:
+
+```powershell
+> ./Update_SRE_RDS_SSL_Certificate.ps1 -dryRun $true
+```
+
+to use the `Let's Encrypt` staging server, which will issue certificates more frequently.
+These certificates will **not** be trusted by your browser, and so should not be used in production.
+````
+
+</details>
+
 <details>
 <summary><strong>Deploy web applications (CoCalc, CodiMD and GitLab)</strong></summary>
 

@@ -3,7 +3,7 @@
 # Deploy an SRE with Microsoft RDS
 
 ```{warning}
-Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.1.0` of the Data Safe Haven.
+Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven.
 ```
 
 These instructions will walk you through deploying a Secure Research Environment (SRE) that uses an existing Safe Haven Management (SHM) environment.
@@ -211,6 +211,10 @@ These certificates will **not** be trusted by your browser, and so should not be
 
 To complete the account setup, follow the instructions for password and MFA setup present in the {ref}`user guide <user_setup_password_mfa>`.
 
+```{warning}
+At present, only phone call identification works correctly with MS RDS. Do not attempt to use the Authenticator app. If you have both the Authenticator and phone call set up as authentication methods, select phone call as the default when intending to use the MS RDS interface.
+```
+
 ### {{nut_and_bolt}} Test the Microsoft RDS remote desktop
 
 - Launch a local web browser on your **deployment machine** and go to `https://<SRE ID>.<safe haven domain>` and log in with the user name and password you set up for the non-privileged user account.

@@ -687,7 +687,7 @@ To test all the above, you will need to act both as the {ref}`role_system_manage
 ```
 
 ```{attention}
-{{white_check_mark}} **Verify that:** software uploaded to the by a non-admin can be read by administrators
+{{white_check_mark}} **Verify that:** software uploaded by a non-admin can be read by administrators
 ```
 
 ```{attention}
-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
+    arrow
     BiocManager
     caret
     csv
@@ Expand Down @@