Merge pull request #1590 from craddm/release-v4.1.0

Updates for Release v4.1.0
alan-turing-institute · Sep 4, 2023 · 51cca07 · 51cca07
2 parents a0a027f + b0a95b9
commit 51cca07
Show file tree

Hide file tree

Showing 29 changed files with 163 additions and 76 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -6,7 +6,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 
 # Set package versions
 ARG AZURE_CLI_VERSION="2.42.0"
-ARG PWSH_VERSION="7.3.2"
+ARG PWSH_VERSION="7.3.6"
 
 # Set up TARGETARCH variable to use to pull the right binaries for the current architecture.
 ARG TARGETARCH

diff --git a/deployment/CheckRequirements.ps1 b/deployment/CheckRequirements.ps1
@@ -8,7 +8,7 @@ param (
 Import-Module $PSScriptRoot/common/Logging -Force -ErrorAction Stop
 
 # Requirements
-$PowershellSupportedVersion = "7.3.2"
+$PowershellSupportedVersion = "7.3.6"
 $ModuleVersionRequired = @{
     "Az.Accounts"                                  = @("ge", "2.11.1")
     "Az.Automation"                                = @("ge", "1.9.0")

diff --git a/deployment/secure_research_desktop/packages/packages-r-cran.list b/deployment/secure_research_desktop/packages/packages-r-cran.list
@@ -1,3 +1,4 @@
+arrow
 BiocManager
 caret
 csv

diff --git a/docs/source/deployment/build_srd_image.md b/docs/source/deployment/build_srd_image.md
@@ -108,7 +108,7 @@ PS> ./Provision_Compute_VM.ps1 -shmId <SHM ID>
 
 ```{note}
 - Although the `./Provision_Compute_VM.ps1` script will finish running in a few minutes, the build itself will take several hours.
-- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/verification/analyse_build.py`.
+- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/monitoring/analyse_build.py`.
 - **NB.** You will need to connect from an approved administrator IP address
 - **NB.** the VM will automatically shutdown at the end of the cloud-init process - if you want to analyse the build after this point, you will need to turn it back on in the `Azure` portal.
 ```

diff --git a/docs/source/deployment/deploy_sre_apache_guacamole.md b/docs/source/deployment/deploy_sre_apache_guacamole.md
@@ -105,6 +105,42 @@ PS> ./Setup_SRE_Guacamole_Servers.ps1 -shmId <SHM ID> -sreId <SRE ID>
 
 </details>
 
+<details>
+<summary><strong>Update SSL certificate</strong></summary>
+
+![Powershell: five minutes](https://img.shields.io/static/v1?style=for-the-badge&logo=powershell&label=local&color=blue&message=five%20minutes) at {{file_folder}} `./deployment/secure_research_environment/setup`
+
+```powershell
+PS> ./Update_SRE_SSL_Certificate.ps1 -shmId <SHM ID> -sreId <SRE ID>
+```
+
+- where `<SHM ID>` is the {ref}`management environment ID <roles_deployer_shm_id>` for this SHM
+- where `<SRE ID>` is the {ref}`secure research environment ID <roles_deployer_sre_id>` for this SRE
+- where `<email>` is an email address that you want to be notified when certificates are close to expiry
+
+```{tip}
+`./Update_SRE_RDS_SSL_Certificate.ps1` should be run again whenever you want to update the certificate for this SRE.
+```
+
+```{caution}
+`Let's Encrypt` will only issue **5 certificates per week** for a particular host (e.g. `rdg-sre-sandbox.project.turingsafehaven.ac.uk`).
+To reduce the number of calls to `Let's Encrypt`, the signed certificates are stored in the Key Vault for easy redeployment.
+For production environments this should usually not be an issue.
+```
+
+````{important}
+If you find yourself frequently redeploying a test environment and hit the `Let's Encrypt` certificate limit, you can can use:
+
+```powershell
+> ./Update_SRE_RDS_SSL_Certificate.ps1 -dryRun $true
+```
+
+to use the `Let's Encrypt` staging server, which will issue certificates more frequently.
+These certificates will **not** be trusted by your browser, and so should not be used in production.
+````
+
+</details>
+
 <details>
 <summary><strong>Deploy web applications (CoCalc, CodiMD and GitLab)</strong></summary>
 

diff --git a/docs/source/deployment/deploy_sre_microsoft_rds.md b/docs/source/deployment/deploy_sre_microsoft_rds.md
@@ -3,7 +3,7 @@
 # Deploy an SRE with Microsoft RDS
 
 ```{warning}
-Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.1.0` of the Data Safe Haven.
+Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven.
 ```
 
 These instructions will walk you through deploying a Secure Research Environment (SRE) that uses an existing Safe Haven Management (SHM) environment.
@@ -211,6 +211,10 @@ These certificates will **not** be trusted by your browser, and so should not be
 
 To complete the account setup, follow the instructions for password and MFA setup present in the {ref}`user guide <user_setup_password_mfa>`.
 
+```{warning}
+At present, only phone call identification works correctly with MS RDS. Do not attempt to use the Authenticator app. If you have both the Authenticator and phone call set up as authentication methods, select phone call as the default when intending to use the MS RDS interface.
+```
+
 ### {{nut_and_bolt}} Test the Microsoft RDS remote desktop
 
 - Launch a local web browser on your **deployment machine** and go to `https://<SRE ID>.<safe haven domain>` and log in with the user name and password you set up for the non-privileged user account.

diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md
@@ -687,7 +687,7 @@ To test all the above, you will need to act both as the {ref}`role_system_manage
 ```
 
 ```{attention}
-{{white_check_mark}} **Verify that:** software uploaded to the by a non-admin can be read by administrators
+{{white_check_mark}} **Verify that:** software uploaded by a non-admin can be read by administrators
 ```
 
 ```{attention}

diff --git a/docs/source/roles/researcher/snippets/06_cocalc.partial.md b/docs/source/roles/researcher/snippets/06_cocalc.partial.md
@@ -1,5 +1,9 @@
 ## {{couple}} Collaborate on code using CoCalc
 
+```{warning}
+Support for `CoCalc` is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven.
+```
+
 `CoCalc` is a collaborative calculation and data science environment.
 It lets you work with others on projects, using `Jupyter`, `LaTeX`, `Octave`, `Python` or `R` in collaborative notebooks.
 

diff --git a/docs/source/roles/researcher/snippets/13_MFA.partial.md b/docs/source/roles/researcher/snippets/13_MFA.partial.md
@@ -147,6 +147,10 @@ This is known as multi-factor authentication (MFA).
 
 #### {{iphone}} Authenticator app registration
 
+```{warning}
+If the SRE you are using will use the Microsoft Remote Desktop interface, do not attempt to use the Authenticator app. At present, only phone call identification works correctly with MS RDS. If you have both the Authenticator and phone call set up as methods, select phone call as the default when intending to use the MS RDS interface.
+```
+
 - If you want to use the Microsoft Authenticator app for MFA (which will work if you have wifi but no phone signal) then click on `+ Add sign-in method` and select `Authenticator app`
 
   ```{image} user_guide/account_setup_mfa_add_authenticator_app.png

diff --git a/docs/source/roles/researcher/user_guide_msrds.md b/docs/source/roles/researcher/user_guide_msrds.md
@@ -3,7 +3,7 @@
 # User Guide: Microsoft Remote Desktop
 
 ```{warning}
-Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.1.0` of the Data Safe Haven.
+Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven.
 ```
 
 ## {{beginner}} Introduction

diff --git a/docs/source/roles/system_manager/manage_data.md b/docs/source/roles/system_manager/manage_data.md
@@ -23,7 +23,7 @@ The following steps show how to generate a temporary write-only upload token tha
 - Click `Networking` under `Settings` and paste the data provider's IP address as one of those allowed under the `Firewall` header, then hit the save icon in the top left
 - From the `Overview` tab, click the link to `Containers` (in the middle of the page)
 - Click `ingress`
-- Click `Shared access signature` under `Settings` and do the following:
+- Click `Shared access tokens` under `Settings` and do the following:
     - Under `Permissions`, check these boxes:
         - `Write`
         - `List`
@@ -70,7 +70,7 @@ The {ref}`role_system_manager` creates a time-limited and IP restricted link to
     - Ensure that the IP address of the person to receive the outputs is listed and enter it if not
 - Click `Containers` under `Data storage`
 - Click `egress`
-- Click `Shared access signature` under `Settings` and do the following:
+- Click `Shared access tokens` under `Settings` and do the following:
     - Under `Permissions`, check these boxes:
         - `Read`
         - `List`

diff --git a/environment_configs/package_lists/allowlist-core-r-cran-tier3.list b/environment_configs/package_lists/allowlist-core-r-cran-tier3.list
@@ -1,3 +1,4 @@
+arrow
 BiocManager
 car
 caret

diff --git a/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list b/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list
@@ -1 +0,0 @@
-arrow

diff --git a/environment_configs/package_lists/allowlist-full-r-cran-tier3.list b/environment_configs/package_lists/allowlist-full-r-cran-tier3.list
@@ -91,6 +91,7 @@ DBI
 dbplyr
 ddalpha
 debugme
+decor
 deldir
 DEoptimR
 desc
@@ -105,6 +106,7 @@ diffobj
 digest
 dimRed
 distributional
+distro
 doMC
 doParallel
 dotCall64
@@ -114,6 +116,7 @@ dplyr
 DRR
 DT
 dtplyr
+duckdb
 dygraphs
 e1071
 ellipsis

diff --git a/environment_configs/sre_bluet1guac_core_config.json b/environment_configs/sre_bluet1guac_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

diff --git a/environment_configs/sre_bluet2guac_core_config.json b/environment_configs/sre_bluet2guac_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

diff --git a/environment_configs/sre_bluet2msrds_core_config.json b/environment_configs/sre_bluet2msrds_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "MicrosoftRDS",
     "dataAdminIpAddresses": ["193.60.220.253"],

diff --git a/environment_configs/sre_bluet3guac_core_config.json b/environment_configs/sre_bluet3guac_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.240"],

diff --git a/environment_configs/sre_bluet3msrds_core_config.json b/environment_configs/sre_bluet3msrds_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "MicrosoftRDS",
     "dataAdminIpAddresses": ["193.60.220.240"],

diff --git a/environment_configs/sre_greent1guac_core_config.json b/environment_configs/sre_greent1guac_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

diff --git a/environment_configs/sre_greent2guac_core_config.json b/environment_configs/sre_greent2guac_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.253"],

diff --git a/environment_configs/sre_greent2msrds_core_config.json b/environment_configs/sre_greent2msrds_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "MicrosoftRDS",
     "dataAdminIpAddresses": ["193.60.220.253"],

diff --git a/environment_configs/sre_greent3guac_core_config.json b/environment_configs/sre_greent3guac_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "ApacheGuacamole",
     "dataAdminIpAddresses": ["193.60.220.240"],

diff --git a/environment_configs/sre_greent3msrds_core_config.json b/environment_configs/sre_greent3msrds_core_config.json
@@ -8,7 +8,7 @@
     "outboundInternetAccess": "default",
     "computeVmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
     },
     "remoteDesktopProvider": "MicrosoftRDS",
     "dataAdminIpAddresses": ["193.60.220.240"],

diff --git a/tests/resources/shm_blue_full_config.json b/tests/resources/shm_blue_full_config.json
@@ -150,6 +150,13 @@
           "217.196.149.55",
           "91.189.91.38",
           "91.189.91.39",
+          "91.189.91.48",
+          "91.189.91.49",
+          "91.189.91.81",
+          "91.189.91.82",
+          "91.189.91.83",
+          "185.125.190.17",
+          "185.125.190.18",
           "185.125.190.36",
           "185.125.190.39",
           "103.21.244.0/22",

diff --git a/tests/resources/shm_green_full_config.json b/tests/resources/shm_green_full_config.json
@@ -150,6 +150,13 @@
           "217.196.149.55",
           "91.189.91.38",
           "91.189.91.39",
+          "91.189.91.48",
+          "91.189.91.49",
+          "91.189.91.81",
+          "91.189.91.82",
+          "91.189.91.83",
+          "185.125.190.17",
+          "185.125.190.18",
           "185.125.190.36",
           "185.125.190.39",
           "103.21.244.0/22",

diff --git a/tests/resources/sre_bluet1guac_full_config.json b/tests/resources/sre_bluet1guac_full_config.json
@@ -151,6 +151,13 @@
             "217.196.149.55",
             "91.189.91.38",
             "91.189.91.39",
+            "91.189.91.48",
+            "91.189.91.49",
+            "91.189.91.81",
+            "91.189.91.82",
+            "91.189.91.83",
+            "185.125.190.17",
+            "185.125.190.18",
             "185.125.190.36",
             "185.125.190.39",
             "103.21.244.0/22",
@@ -1111,15 +1118,8 @@
       "instances": [
         {
           "adminPasswordSecretName": "sre-t1guac-vm-admin-password-mssql",
-          "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql",
           "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-mssql",
-          "vmName": "MSSQL-T1GUAC",
-          "type": "MSSQL",
-          "ip": "10.151.3.4",
-          "port": "1433",
-          "sku": "sqldev-gen2",
-          "subnet": "databases",
-          "vmSize": "Standard_DS2_v2",
+          "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql",
           "disks": {
             "data": {
               "sizeGb": "1024",
@@ -1130,19 +1130,19 @@
               "type": "Standard_LRS"
             }
           },
-          "enableSSIS": true
+          "enableSSIS": true,
+          "ip": "10.151.3.4",
+          "port": "1433",
+          "sku": "sqldev-gen2",
+          "subnet": "databases",
+          "type": "MSSQL",
+          "vmName": "MSSQL-T1GUAC",
+          "vmSize": "Standard_DS2_v2"
         },
         {
           "adminPasswordSecretName": "sre-t1guac-vm-admin-password-postgresql",
-          "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql",
           "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-postgresql",
-          "vmName": "PSTGRS-T1GUAC",
-          "type": "PostgreSQL",
-          "ip": "10.151.3.5",
-          "port": "5432",
-          "sku": "Ubuntu-latest",
-          "subnet": "databases",
-          "vmSize": "Standard_DS2_v2",
+          "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql",
           "disks": {
             "data": {
               "sizeGb": "1024",
@@ -1152,7 +1152,14 @@
               "sizeGb": "128",
               "type": "Standard_LRS"
             }
-          }
+          },
+          "ip": "10.151.3.5",
+          "port": "5432",
+          "sku": "Ubuntu-latest",
+          "subnet": "databases",
+          "type": "PostgreSQL",
+          "vmName": "PSTGRS-T1GUAC",
+          "vmSize": "Standard_DS2_v2"
         }
       ],
       "rg": "RG_SHM_BLUE_SRE_T1GUAC_DATABASES"
@@ -1302,7 +1309,7 @@
       "rg": "RG_SHM_BLUE_SRE_T1GUAC_COMPUTE",
       "vmImage": {
         "type": "Ubuntu",
-        "version": "20.04.2023031401"
+        "version": "20.04.2023082900"
       },
       "vmSizeDefault": "Standard_D2s_v3"
     },
-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
+    arrow
     BiocManager
     caret
     csv
@@ Expand Down @@