From ddc018fd77a35620fa1df7994dd5a5f0017bf9b5 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Tue, 25 Jul 2023 13:15:35 +0100 Subject: [PATCH 01/33] :recycle: Replace poetry with pip-compile for better dependency solving --- .../cloud_init/resources/pyenv_install.sh | 23 ++++++++----------- .../packages/packages-python.yaml | 3 +-- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh b/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh index 555c46c14f..2770f82466 100644 --- a/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh +++ b/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh @@ -27,24 +27,19 @@ echo "Installed $(${EXE_PATH}/python --version)" # Install and upgrade installation prerequisites # ---------------------------------------------- echo "Installing and upgrading installation prerequisites for Python ${PYTHON_VERSION}..." -${EXE_PATH}/pip install --upgrade pip poetry +${EXE_PATH}/pip install --upgrade pip pip-tools setuptools -# Solve dependencies and install using poetry -# ------------------------------------------- -echo "Installing packages with poetry..." -${EXE_PATH}/poetry config virtualenvs.create false -${EXE_PATH}/poetry config virtualenvs.in-project true -rm poetry.lock pyproject.toml 2> /dev/null -sed -e "s/PYTHON_VERSION/$PYTHON_VERSION/" /opt/build/pyenv/pyproject_template.toml > $PYPROJECT_TOML -ln -s $PYPROJECT_TOML pyproject.toml -${EXE_PATH}/poetry add $(tr '\n' ' ' < $REQUIREMENTS_TXT) || exit 3 +# Solve dependencies and write package versions to monitoring log +# --------------------------------------------------------------- +echo "Determining package versions with pip-compile..." +${EXE_PATH}/pip-compile -o "$MONITORING_LOG" "$REQUIREMENTS_TXT" -# Write package versions to monitoring log -# ---------------------------------------- -${EXE_PATH}/poetry show > $MONITORING_LOG -${EXE_PATH}/poetry show --tree >> $MONITORING_LOG +# Install pinned packages using pip +# --------------------------------- +echo "Installing packages with pip..." +${EXE_PATH}/pip install -r "$MONITORING_LOG" # Run any post-install commands diff --git a/deployment/secure_research_desktop/packages/packages-python.yaml b/deployment/secure_research_desktop/packages/packages-python.yaml index 879788d373..d520b3d60e 100644 --- a/deployment/secure_research_desktop/packages/packages-python.yaml +++ b/deployment/secure_research_desktop/packages/packages-python.yaml @@ -39,9 +39,8 @@ packages: pathos: pg8000: Pillow: + pip-tools: plotly: - poetry: # also used by installation scripts - "all": [">1.0.0"] # increase solver flexibility prophet: psycopg2: pydot: From b176e94cdbdceb0e6c49407191bd8271c637f397 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Tue, 25 Jul 2023 13:15:00 +0100 Subject: [PATCH 02/33] :coffin: Remove pyproject.toml --- .../cloud-init-buildimage-ubuntu-1804.mustache.yaml | 5 ----- .../cloud-init-buildimage-ubuntu-2004.mustache.yaml | 5 ----- .../cloud-init-buildimage-ubuntu-2204.mustache.yaml | 5 ----- .../cloud_init/resources/pyenv_install.sh | 1 - .../resources/pyenv_pyproject_template.toml | 12 ------------ 5 files changed, 28 deletions(-) delete mode 100644 deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml index 44e1bb4397..8f450ead3d 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml @@ -114,11 +114,6 @@ write_files: content: | {{packages-python.yaml}} - - path: "/opt/build/pyenv/pyproject_template.toml" - permissions: "0400" - content: | - {{pyenv_pyproject_template.toml}} - - path: "/opt/build/rstudio.debinfo" permissions: "0400" content: | diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml index 5605e07707..575de168d0 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml @@ -114,11 +114,6 @@ write_files: content: | {{packages-python.yaml}} - - path: "/opt/build/pyenv/pyproject_template.toml" - permissions: "0400" - content: | - {{pyenv_pyproject_template.toml}} - - path: "/opt/build/rstudio.debinfo" permissions: "0400" content: | diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml index 17527746cb..059dece327 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml @@ -118,11 +118,6 @@ write_files: content: | {{packages-python.yaml}} - - path: "/opt/build/pyenv/pyproject_template.toml" - permissions: "0400" - content: | - {{pyenv_pyproject_template.toml}} - - path: "/opt/build/rbase.debinfo" permissions: "0400" content: | diff --git a/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh b/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh index 2770f82466..56898d3915 100644 --- a/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh +++ b/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh @@ -9,7 +9,6 @@ if [ $# -ne 1 ]; then fi PYTHON_VERSION=$1 PYENV_ROOT="$(pyenv root)" -PYPROJECT_TOML="/opt/build/python-${PYTHON_VERSION}-pyproject.toml" MONITORING_LOG="/opt/monitoring/python-${PYTHON_VERSION}-package-versions.log" REQUIREMENTS_TXT="/opt/build/python-${PYTHON_VERSION}-requirements.txt" REQUESTED_PACKAGE_LIST="/opt/build/packages/packages-python-${PYTHON_VERSION}.list" diff --git a/deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml b/deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml deleted file mode 100644 index 3f0998952a..0000000000 --- a/deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml +++ /dev/null @@ -1,12 +0,0 @@ -[tool.poetry] -name = "Python PYTHON_VERSION" -version = "1.0.0" -description = "Python PYTHON_VERSION" -authors = ["ROOT "] - -[tool.poetry.dependencies] -python = "PYTHON_VERSION" - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" From 60738a9a1b8734d145e9b2acb3b1cf7ed1494097 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Fri, 4 Aug 2023 10:53:09 +0100 Subject: [PATCH 03/33] add pip-tools to NON_IMPORTABLE_PACKAGES --- tests/srd_smoke_tests/test_packages_installed_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/srd_smoke_tests/test_packages_installed_python.py b/tests/srd_smoke_tests/test_packages_installed_python.py index a01a2c44c1..28cb1538b7 100644 --- a/tests/srd_smoke_tests/test_packages_installed_python.py +++ b/tests/srd_smoke_tests/test_packages_installed_python.py @@ -22,7 +22,7 @@ ] # For these packages we check for an executable as they are not importable -NON_IMPORTABLE_PACKAGES = {"repro-catalogue": "catalogue"} +NON_IMPORTABLE_PACKAGES = {"repro-catalogue": "catalogue", "pip-tools": "pip-compile"} # Some packages are imported using a different name than they `pip install` with IMPORTABLE_NAMES = { From a9b51439a7f5abc1cc25711ca46081cc6e83cc61 Mon Sep 17 00:00:00 2001 From: James Robinson Date: Fri, 4 Aug 2023 11:16:31 +0100 Subject: [PATCH 04/33] :art: Alphabetise NON_IMPORTABLE_PACKAGES --- tests/srd_smoke_tests/test_packages_installed_python.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/srd_smoke_tests/test_packages_installed_python.py b/tests/srd_smoke_tests/test_packages_installed_python.py index 28cb1538b7..d91d3238f7 100644 --- a/tests/srd_smoke_tests/test_packages_installed_python.py +++ b/tests/srd_smoke_tests/test_packages_installed_python.py @@ -4,6 +4,7 @@ import subprocess import sys import warnings + import pkg_resources versions = { @@ -22,7 +23,10 @@ ] # For these packages we check for an executable as they are not importable -NON_IMPORTABLE_PACKAGES = {"repro-catalogue": "catalogue", "pip-tools": "pip-compile"} +NON_IMPORTABLE_PACKAGES = { + "pip-tools": "pip-compile", + "repro-catalogue": "catalogue", +} # Some packages are imported using a different name than they `pip install` with IMPORTABLE_NAMES = { From fc256acce8c53140bd3f4d1c73a4af11c66760dc Mon Sep 17 00:00:00 2001 From: James Robinson Date: Mon, 7 Aug 2023 15:45:32 +0100 Subject: [PATCH 05/33] Update VERSIONING.md Add May 2023 DSG to versioning --- VERSIONING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/VERSIONING.md b/VERSIONING.md index 203a10a1ff..e072045b15 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -64,6 +64,7 @@ We usually deploy the latest available version of the Data Safe Haven for each o | December 2021 | DSG 2021-12 | [v3.3.1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.3.1) | | December 2022 | DSG 2022-12 | [v4.0.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.2) | | February 2023 | DSG 2023-02 | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3) | +| May 2023 | DSG 2023-05 | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3) | ## Versions that have undergone formal security evaluation From 099dc79935461cc1aa75b8b8c227c0cfe6d67523 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:04:58 +0100 Subject: [PATCH 06/33] update signing key gitlab --- .../cloud_init/cloud-init-gitlab.mustache.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml index 8eefcfcbf6..00a48d64ad 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml @@ -93,7 +93,7 @@ apt: sources: gitlab.list: source: "deb https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu focal main" - keyid: F6403F6544A38863DAA0B6E03F01618A51312F3F # GitLab B.V. (package repository signing key) + keyid: DBEF89774DDB9EB37D9FC3A03CFCF9BAF27EAB47 # GitLab B.V. (package repository signing key) # Install necessary apt packages packages: From eb8d5336845038511cde7dea7bf88f6e7756f1ce Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:23:49 +0100 Subject: [PATCH 07/33] revert prev commit --- .../cloud_init/cloud-init-gitlab.mustache.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml index 00a48d64ad..8eefcfcbf6 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml @@ -93,7 +93,7 @@ apt: sources: gitlab.list: source: "deb https://packages.gitlab.com/gitlab/gitlab-ce/ubuntu focal main" - keyid: DBEF89774DDB9EB37D9FC3A03CFCF9BAF27EAB47 # GitLab B.V. (package repository signing key) + keyid: F6403F6544A38863DAA0B6E03F01618A51312F3F # GitLab B.V. (package repository signing key) # Install necessary apt packages packages: From 8a0bccb4e7330cd43492152e0b3b939614577328 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:24:15 +0100 Subject: [PATCH 08/33] change fs_setup partition to auto --- .../cloud_init/cloud-init-gitlab.mustache.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml index 8eefcfcbf6..b887420e2b 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml @@ -122,7 +122,7 @@ disk_setup: overwrite: true fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"] From 08c9a6f488b70eb6a6da1dcfe4dcbba631b821e1 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:25:37 +0100 Subject: [PATCH 09/33] same change for cocalc and codimd --- .../cloud_init/cloud-init-cocalc.mustache.yaml | 2 +- .../cloud_init/cloud-init-codimd.mustache.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml index 43da1b27da..3b43fd408c 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml @@ -161,7 +161,7 @@ disk_setup: overwrite: true fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"] diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml index f2afe406c1..890c39d5e3 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml @@ -132,7 +132,7 @@ disk_setup: overwrite: true fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"] From a1735a0eeaa1900c3782f6cee4e21dbedf103413 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:43:22 +0100 Subject: [PATCH 10/33] add *.docker.io to allowedFqdns --- .../setup/Configure_External_DNS_Queries.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 index a86077d1d1..fbe41fc760 100644 --- a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 +++ b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 @@ -26,7 +26,8 @@ $null = Set-AzContext -SubscriptionId $config.sre.subscriptionName -ErrorAction # -------------------------------------- $firewallRules = Get-JsonFromMustacheTemplate -TemplatePath (Join-Path $PSScriptRoot ".." ".." "safe_haven_management_environment" "network_rules" "shm-firewall-rules.json") -Parameters $config.shm -AsHashtable $allowedFqdns = @($firewallRules.applicationRuleCollections | ForEach-Object { $_.properties.rules.targetFqdns }) + - @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name }) + @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name }) + + @("*.docker.io") # List all unique FQDNs $allowedFqdns = $allowedFqdns | Where-Object { $_ -notlike "*-sb.servicebus.windows.net" } | # Remove AzureADConnect password reset endpoints From a414373406920f80dfb13e83b55fec641e708ba1 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:53:33 +0100 Subject: [PATCH 11/33] change partition to auto for remaining cloud-inits --- .../cloud_init/cloud-init-postgres.mustache.yaml | 2 +- .../cloud_init/cloud-init-srd.mustache.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml index 16bcb94fa9..2a6c8f707b 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml @@ -25,7 +25,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml index 12065238e6..3635a7a342 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml @@ -12,7 +12,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 filesystem: ext4 - partition: 1 + partition: auto # Note that we do not include the blobfuse mounts here as these are controlled by systemd mounts: From 0be00893f2351dcffb955b871bd1a72ef0332003 Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:55:24 +0100 Subject: [PATCH 12/33] change fs_setup partition to auto cloud init shm --- .../cloud-init-repository-mirror-external-cran.mustache.yaml | 2 +- .../cloud-init-repository-mirror-external-pypi.mustache.yaml | 2 +- .../cloud-init-repository-mirror-internal-cran.mustache.yaml | 2 +- .../cloud-init-repository-mirror-internal-pypi.mustache.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml index 38b7e3cb87..9bcd26b283 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml index 8c09932601..14f71356f5 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml index 9f0a108f60..6c67f7d4dd 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml index 2fad1ab1ee..3507b51dbd 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: From 17a0900de9663c6dd570b855646bf42f33c01aba Mon Sep 17 00:00:00 2001 From: Ed Chalstrey Date: Wed, 9 Aug 2023 10:58:23 +0100 Subject: [PATCH 13/33] change *.docker.io to docker.io --- .../setup/Configure_External_DNS_Queries.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 index fbe41fc760..51558b00b5 100644 --- a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 +++ b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 @@ -27,7 +27,7 @@ $null = Set-AzContext -SubscriptionId $config.sre.subscriptionName -ErrorAction $firewallRules = Get-JsonFromMustacheTemplate -TemplatePath (Join-Path $PSScriptRoot ".." ".." "safe_haven_management_environment" "network_rules" "shm-firewall-rules.json") -Parameters $config.shm -AsHashtable $allowedFqdns = @($firewallRules.applicationRuleCollections | ForEach-Object { $_.properties.rules.targetFqdns }) + @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name }) + - @("*.docker.io") + @("docker.io") # List all unique FQDNs $allowedFqdns = $allowedFqdns | Where-Object { $_ -notlike "*-sb.servicebus.windows.net" } | # Remove AzureADConnect password reset endpoints From 154a55a9a671c762efe32e3121a1fbddc1f83f5d Mon Sep 17 00:00:00 2001 From: James Robinson Date: Wed, 9 Aug 2023 15:05:11 +0100 Subject: [PATCH 14/33] :wrench: Add additional *.ubuntu.com IP addresses --- deployment/common/Configuration.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/common/Configuration.psm1 b/deployment/common/Configuration.psm1 index c1090d9bb8..a77c9416d3 100644 --- a/deployment/common/Configuration.psm1 +++ b/deployment/common/Configuration.psm1 @@ -332,7 +332,7 @@ function Get-ShmConfig { ) # *-jobruntimedata-prod-su1.azure-automation.net linux = ( @("72.32.157.246", "87.238.57.227", "147.75.85.69", "217.196.149.55") + # apt.postgresql.org - @("91.189.91.38", "91.189.91.39", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com + @("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com $cloudFlareIpAddresses + # database.clamav.net, packages.gitlab.com and qgis.org use Cloudflare $cloudFrontIpAddresses + # packages.gitlab.com uses Cloudfront to host its Release file @("104.131.190.124") + # dbeaver.io From 0ac99dee46b633a6d5ed661bfd85f91c195af861 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 29 Aug 2023 09:51:37 +0000 Subject: [PATCH 15/33] Pin packaging package version and fix rsync typo --- .../cloud-init-repository-mirror-external-pypi.mustache.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml index 14f71356f5..0b38914c3b 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml @@ -122,7 +122,7 @@ write_files: - path: "/etc/cron.d/pull-from-internet" permissions: "0644" content: | - # External update (rsync from CRAN) every 6 hours + # External update from PyPi every 6 hours 0 */6 * * * mirrordaemon ~mirrordaemon/pull_from_internet.sh - path: "/etc/cron.d/pull-then-push" @@ -313,7 +313,7 @@ runcmd: # Install bandersnatch with pip - echo ">=== Installing bandersnatch... ===<" - - pip3 install bandersnatch==4.2.0 + - pip3 install bandersnatch==4.2.0 packaging==21.3 - echo "Using bandersnatch from '$(which bandersnatch)'" # Initialise allowlist if appropriate From 6be3be07e5575a6a0ea0758fcaa43b91202831fc Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:42:46 +0000 Subject: [PATCH 16/33] Force MSSQL to trust server certificate --- .../create_databases/scripts/Lockdown_Sql_Server.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 b/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 index a47ddba25b..101c9f2f7b 100644 --- a/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 +++ b/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 @@ -107,7 +107,7 @@ if ($operationFailed -Or (-Not $loginExists)) { # Create a DB user for each login group Write-Output "Ensuring that an SQL user exists for '$domainGroup' on: '$serverName'..." $sqlCommand = "IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$domainGroup') CREATE USER [$domainGroup] FOR LOGIN [$domainGroup];" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query "$sqlCommand" -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Ensured that '$domainGroup' user exists on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -124,7 +124,7 @@ if ($operationFailed -Or (-Not $loginExists)) { foreach ($groupSchemaTuple in @(($DataAdminGroup, "data"), ($ResearchUsersGroup, "dbopublic"))) { $domainGroup, $schemaName = $groupSchemaTuple $sqlCommand = "IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = N'$schemaName') EXEC('CREATE SCHEMA $schemaName AUTHORIZATION [$domainGroup]');" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully ensured that '$schemaName' schema exists on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -154,7 +154,7 @@ if ($operationFailed -Or (-Not $loginExists)) { Write-Output " [x] Role $role not recognised!" continue } - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully gave '$domainGroup' $role permissions on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -171,7 +171,7 @@ if ($operationFailed -Or (-Not $loginExists)) { # ------------------------------------ Write-Output "Running T-SQL lockdown script on: '$serverName'..." $sqlCommand = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ServerLockdownCommandB64)) - Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully ran T-SQL lockdown script on: '$serverName'" } else { @@ -187,7 +187,7 @@ if ($operationFailed -Or (-Not $loginExists)) { $windowsAdmin = "${serverName}\${VmAdminUsername}" Write-Output "Removing database access from $windowsAdmin on: '$serverName'..." $sqlCommand = "DROP USER IF EXISTS [$windowsAdmin]; IF EXISTS(SELECT * FROM master.dbo.syslogins WHERE loginname = '$windowsAdmin') DROP LOGIN [$windowsAdmin]" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully removed database access for $windowsAdmin on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -203,7 +203,7 @@ if ($operationFailed -Or (-Not $loginExists)) { # --------------------------------------------------------------------------------- Write-Output "Revoking sysadmin role from $DbAdminUsername on: '$serverName'..." $sqlCommand = "ALTER SERVER ROLE sysadmin DROP MEMBER $DbAdminUsername;" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully revoked sysadmin role on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete From 6909c8c1d1c68c2b3b4da1f8ebd8c3b048d70955 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 29 Aug 2023 11:45:51 +0000 Subject: [PATCH 17/33] Always try to redeploy existing MSSQL server --- .../secure_research_environment/setup/Setup_SRE_Databases.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 b/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 index 0b7fbb2361..c7e8586193 100644 --- a/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 +++ b/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 @@ -58,7 +58,6 @@ foreach ($databaseCfg in $config.sre.databases.instances) { } } else { Add-LogMessage -Level Warning "Database VM '$($databaseCfg.vmName)' already exists. Use the '-Redeploy' option if you want to remove the existing database and its data and deploy a new one." - continue } } From c677c2d05593ac55a8a22032eae0dae688f18012 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 08:52:17 +0000 Subject: [PATCH 18/33] Update CRAN packagelists to include arrow --- .../secure_research_desktop/packages/packages-r-cran.list | 1 + .../package_lists/allowlist-core-r-cran-tier3.list | 1 + .../package_lists/allowlist-extra-r-cran-tier3.list | 1 - .../package_lists/allowlist-full-r-cran-tier3.list | 3 +++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/deployment/secure_research_desktop/packages/packages-r-cran.list b/deployment/secure_research_desktop/packages/packages-r-cran.list index 0c716553bb..7cf7e374a6 100644 --- a/deployment/secure_research_desktop/packages/packages-r-cran.list +++ b/deployment/secure_research_desktop/packages/packages-r-cran.list @@ -1,3 +1,4 @@ +arrow BiocManager caret csv diff --git a/environment_configs/package_lists/allowlist-core-r-cran-tier3.list b/environment_configs/package_lists/allowlist-core-r-cran-tier3.list index 23977ee481..7fc527f500 100644 --- a/environment_configs/package_lists/allowlist-core-r-cran-tier3.list +++ b/environment_configs/package_lists/allowlist-core-r-cran-tier3.list @@ -1,3 +1,4 @@ +arrow BiocManager car caret diff --git a/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list b/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list index e2dc7471c2..e69de29bb2 100644 --- a/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list +++ b/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list @@ -1 +0,0 @@ -arrow diff --git a/environment_configs/package_lists/allowlist-full-r-cran-tier3.list b/environment_configs/package_lists/allowlist-full-r-cran-tier3.list index 9e12103115..7e629c72b2 100644 --- a/environment_configs/package_lists/allowlist-full-r-cran-tier3.list +++ b/environment_configs/package_lists/allowlist-full-r-cran-tier3.list @@ -91,6 +91,7 @@ DBI dbplyr ddalpha debugme +decor deldir DEoptimR desc @@ -105,6 +106,7 @@ diffobj digest dimRed distributional +distro doMC doParallel dotCall64 @@ -114,6 +116,7 @@ dplyr DRR DT dtplyr +duckdb dygraphs e1071 ellipsis From 481ad4097acca92fe6e8e0c5ab25b8c84659558e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:06:03 +0000 Subject: [PATCH 19/33] Fix filepath of SRD build summary script in docs --- docs/source/deployment/build_srd_image.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment/build_srd_image.md b/docs/source/deployment/build_srd_image.md index 55e4e769e5..9482988160 100644 --- a/docs/source/deployment/build_srd_image.md +++ b/docs/source/deployment/build_srd_image.md @@ -108,7 +108,7 @@ PS> ./Provision_Compute_VM.ps1 -shmId ```{note} - Although the `./Provision_Compute_VM.ps1` script will finish running in a few minutes, the build itself will take several hours. -- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/verification/analyse_build.py`. +- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/monitoring/analyse_build.py`. - **NB.** You will need to connect from an approved administrator IP address - **NB.** the VM will automatically shutdown at the end of the cloud-init process - if you want to analyse the build after this point, you will need to turn it back on in the `Azure` portal. ``` From c385aa7e46fd20aabb30fdcd3ee6495fe4ae6dc4 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:06:34 +0000 Subject: [PATCH 20/33] Increase MSRDS deprecation warning version number --- docs/source/deployment/deploy_sre_microsoft_rds.md | 2 +- docs/source/roles/researcher/user_guide_msrds.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/deployment/deploy_sre_microsoft_rds.md b/docs/source/deployment/deploy_sre_microsoft_rds.md index 40a2035c3a..a850afe52a 100644 --- a/docs/source/deployment/deploy_sre_microsoft_rds.md +++ b/docs/source/deployment/deploy_sre_microsoft_rds.md @@ -3,7 +3,7 @@ # Deploy an SRE with Microsoft RDS ```{warning} -Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.1.0` of the Data Safe Haven. +Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven. ``` These instructions will walk you through deploying a Secure Research Environment (SRE) that uses an existing Safe Haven Management (SHM) environment. diff --git a/docs/source/roles/researcher/user_guide_msrds.md b/docs/source/roles/researcher/user_guide_msrds.md index 67d76c6270..263bf56650 100644 --- a/docs/source/roles/researcher/user_guide_msrds.md +++ b/docs/source/roles/researcher/user_guide_msrds.md @@ -3,7 +3,7 @@ # User Guide: Microsoft Remote Desktop ```{warning} -Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.1.0` of the Data Safe Haven. +Support for Microsoft Remote Desktop is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven. ``` ## {{beginner}} Introduction From 413a9800d54df0314c5802146f3614e2ab1755d8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:13:17 +0000 Subject: [PATCH 21/33] Add cocalc deprecation warning --- docs/source/roles/researcher/snippets/06_cocalc.partial.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/roles/researcher/snippets/06_cocalc.partial.md b/docs/source/roles/researcher/snippets/06_cocalc.partial.md index 7940aa7f2b..79b9af5b2b 100644 --- a/docs/source/roles/researcher/snippets/06_cocalc.partial.md +++ b/docs/source/roles/researcher/snippets/06_cocalc.partial.md @@ -1,5 +1,9 @@ ## {{couple}} Collaborate on code using CoCalc +```{warning} +Support for `CoCalc` is deprecated. Deployment scripts and related documentation will be removed in version `4.2.0` of the Data Safe Haven. +``` + `CoCalc` is a collaborative calculation and data science environment. It lets you work with others on projects, using `Jupyter`, `LaTeX`, `Octave`, `Python` or `R` in collaborative notebooks. From 470128189323c430d677eddf09b7e37d68304836 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:16:13 +0000 Subject: [PATCH 22/33] add ssl certificate step to guac deployment docs --- .../deployment/deploy_sre_apache_guacamole.md | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/source/deployment/deploy_sre_apache_guacamole.md b/docs/source/deployment/deploy_sre_apache_guacamole.md index 9f5f69533e..94403084fc 100644 --- a/docs/source/deployment/deploy_sre_apache_guacamole.md +++ b/docs/source/deployment/deploy_sre_apache_guacamole.md @@ -105,6 +105,42 @@ PS> ./Setup_SRE_Guacamole_Servers.ps1 -shmId -sreId +
+Update SSL certificate + +![Powershell: five minutes](https://img.shields.io/static/v1?style=for-the-badge&logo=powershell&label=local&color=blue&message=five%20minutes) at {{file_folder}} `./deployment/secure_research_environment/setup` + +```powershell +PS> ./Update_SRE_SSL_Certificate.ps1 -shmId -sreId +``` + +- where `` is the {ref}`management environment ID ` for this SHM +- where `` is the {ref}`secure research environment ID ` for this SRE +- where `` is an email address that you want to be notified when certificates are close to expiry + +```{tip} +`./Update_SRE_RDS_SSL_Certificate.ps1` should be run again whenever you want to update the certificate for this SRE. +``` + +```{caution} +`Let's Encrypt` will only issue **5 certificates per week** for a particular host (e.g. `rdg-sre-sandbox.project.turingsafehaven.ac.uk`). +To reduce the number of calls to `Let's Encrypt`, the signed certificates are stored in the Key Vault for easy redeployment. +For production environments this should usually not be an issue. +``` + +````{important} +If you find yourself frequently redeploying a test environment and hit the `Let's Encrypt` certificate limit, you can can use: + +```powershell +> ./Update_SRE_RDS_SSL_Certificate.ps1 -dryRun $true +``` + +to use the `Let's Encrypt` staging server, which will issue certificates more frequently. +These certificates will **not** be trusted by your browser, and so should not be used in production. +```` + +
+
Deploy web applications (CoCalc, CodiMD and GitLab) From 09d9c1be07166a7bf808953a857e4d5eb8430af7 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:16:26 +0000 Subject: [PATCH 23/33] update dockerfile to 7.3.6 --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 85ab92fd66..4a852d8499 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Set package versions ARG AZURE_CLI_VERSION="2.42.0" -ARG PWSH_VERSION="7.3.2" +ARG PWSH_VERSION="7.3.6" # Set up TARGETARCH variable to use to pull the right binaries for the current architecture. ARG TARGETARCH From 75ae395cfed35581f0d7d1f3c1f1abc433cda5a0 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 30 Aug 2023 10:57:10 +0000 Subject: [PATCH 24/33] Update core configs with new vm image --- environment_configs/sre_greent2msrds_core_config.json | 2 +- environment_configs/sre_greent3guac_core_config.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment_configs/sre_greent2msrds_core_config.json b/environment_configs/sre_greent2msrds_core_config.json index 20c6912ec9..b5ae22cd47 100644 --- a/environment_configs/sre_greent2msrds_core_config.json +++ b/environment_configs/sre_greent2msrds_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "MicrosoftRDS", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent3guac_core_config.json b/environment_configs/sre_greent3guac_core_config.json index 95f59302f5..a3c0c4c57e 100644 --- a/environment_configs/sre_greent3guac_core_config.json +++ b/environment_configs/sre_greent3guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.240"], From 00b03c5c09213ecbbd1f66c8f00c5e52904d4a3b Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:57:19 +0000 Subject: [PATCH 25/33] Update supported Powershell version --- deployment/CheckRequirements.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/CheckRequirements.ps1 b/deployment/CheckRequirements.ps1 index 740f081581..6fad3f01c4 100644 --- a/deployment/CheckRequirements.ps1 +++ b/deployment/CheckRequirements.ps1 @@ -8,7 +8,7 @@ param ( Import-Module $PSScriptRoot/common/Logging -Force -ErrorAction Stop # Requirements -$PowershellSupportedVersion = "7.3.2" +$PowershellSupportedVersion = "7.3.6" $ModuleVersionRequired = @{ "Az.Accounts" = @("ge", "2.11.1") "Az.Automation" = @("ge", "1.9.0") From 067d75c7d24da46d4ca6a21dd18126ec73fd9cb7 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:04:11 +0000 Subject: [PATCH 26/33] Fix typo --- docs/source/deployment/security_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md index f8dcc553c6..07e9339f2e 100644 --- a/docs/source/deployment/security_checklist.md +++ b/docs/source/deployment/security_checklist.md @@ -687,7 +687,7 @@ To test all the above, you will need to act both as the {ref}`role_system_manage ``` ```{attention} -{{white_check_mark}} **Verify that:** software uploaded to the by a non-admin can be read by administrators +{{white_check_mark}} **Verify that:** software uploaded by a non-admin can be read by administrators ``` ```{attention} From 51e35ac251cc35b887fe45ef8210061d46e707d8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:04:36 +0000 Subject: [PATCH 27/33] Update text to reflect changes in Portal --- docs/source/roles/system_manager/manage_data.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/roles/system_manager/manage_data.md b/docs/source/roles/system_manager/manage_data.md index e517821476..9ebd6013d3 100644 --- a/docs/source/roles/system_manager/manage_data.md +++ b/docs/source/roles/system_manager/manage_data.md @@ -23,7 +23,7 @@ The following steps show how to generate a temporary write-only upload token tha - Click `Networking` under `Settings` and paste the data provider's IP address as one of those allowed under the `Firewall` header, then hit the save icon in the top left - From the `Overview` tab, click the link to `Containers` (in the middle of the page) - Click `ingress` -- Click `Shared access signature` under `Settings` and do the following: +- Click `Shared access tokens` under `Settings` and do the following: - Under `Permissions`, check these boxes: - `Write` - `List` @@ -70,7 +70,7 @@ The {ref}`role_system_manager` creates a time-limited and IP restricted link to - Ensure that the IP address of the person to receive the outputs is listed and enter it if not - Click `Containers` under `Data storage` - Click `egress` -- Click `Shared access signature` under `Settings` and do the following: +- Click `Shared access tokens` under `Settings` and do the following: - Under `Permissions`, check these boxes: - `Read` - `List` From 0c1597abf7affd7baf9b91ab1f5624208fd31b64 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:10:23 +0000 Subject: [PATCH 28/33] Add warning to useonly phone call auth with MSRDS --- docs/source/roles/researcher/snippets/13_MFA.partial.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/roles/researcher/snippets/13_MFA.partial.md b/docs/source/roles/researcher/snippets/13_MFA.partial.md index bc4812d709..2af52825b3 100644 --- a/docs/source/roles/researcher/snippets/13_MFA.partial.md +++ b/docs/source/roles/researcher/snippets/13_MFA.partial.md @@ -147,6 +147,10 @@ This is known as multi-factor authentication (MFA). #### {{iphone}} Authenticator app registration +```{warning} +If the SRE you are using will use the Microsoft Remote Desktop interface, do not attempt to use the Authenticator app. At present, only phone call identification works correctly with MS RDS. If you have both the Authenticator and phone call set up as methods, select phone call as the default when intending to use the MS RDS interface. +``` + - If you want to use the Microsoft Authenticator app for MFA (which will work if you have wifi but no phone signal) then click on `+ Add sign-in method` and select `Authenticator app` ```{image} user_guide/account_setup_mfa_add_authenticator_app.png From 5d3717f7e2e1c145a0f2f24f9645e6038cfd0d60 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 1 Sep 2023 12:12:02 +0000 Subject: [PATCH 29/33] Add warning about MS RDS auth method --- docs/source/deployment/deploy_sre_microsoft_rds.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/deployment/deploy_sre_microsoft_rds.md b/docs/source/deployment/deploy_sre_microsoft_rds.md index a850afe52a..3ea0cd14e2 100644 --- a/docs/source/deployment/deploy_sre_microsoft_rds.md +++ b/docs/source/deployment/deploy_sre_microsoft_rds.md @@ -211,6 +211,10 @@ These certificates will **not** be trusted by your browser, and so should not be To complete the account setup, follow the instructions for password and MFA setup present in the {ref}`user guide `. +```{warning} +At present, only phone call identification works correctly with MS RDS. Do not attempt to use the Authenticator app. If you have both the Authenticator and phone call set up as authentication methods, select phone call as the default when intending to use the MS RDS interface. +``` + ### {{nut_and_bolt}} Test the Microsoft RDS remote desktop - Launch a local web browser on your **deployment machine** and go to `https://.` and log in with the user name and password you set up for the non-privileged user account. From 3eadac7c83dc995f9f7d4b3b69f1849d5d3cbe23 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 1 Sep 2023 13:47:53 +0000 Subject: [PATCH 30/33] Update core configs to use latest SRD image --- environment_configs/sre_bluet1guac_core_config.json | 2 +- environment_configs/sre_bluet2guac_core_config.json | 2 +- environment_configs/sre_bluet2msrds_core_config.json | 2 +- environment_configs/sre_bluet3guac_core_config.json | 2 +- environment_configs/sre_bluet3msrds_core_config.json | 2 +- environment_configs/sre_greent1guac_core_config.json | 2 +- environment_configs/sre_greent2guac_core_config.json | 2 +- environment_configs/sre_greent3msrds_core_config.json | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/environment_configs/sre_bluet1guac_core_config.json b/environment_configs/sre_bluet1guac_core_config.json index 2454eb73e8..365ae14a8c 100644 --- a/environment_configs/sre_bluet1guac_core_config.json +++ b/environment_configs/sre_bluet1guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet2guac_core_config.json b/environment_configs/sre_bluet2guac_core_config.json index 2ada45f6ec..15d96d2cda 100644 --- a/environment_configs/sre_bluet2guac_core_config.json +++ b/environment_configs/sre_bluet2guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet2msrds_core_config.json b/environment_configs/sre_bluet2msrds_core_config.json index 13e8a4e218..f5174dc929 100644 --- a/environment_configs/sre_bluet2msrds_core_config.json +++ b/environment_configs/sre_bluet2msrds_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "MicrosoftRDS", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet3guac_core_config.json b/environment_configs/sre_bluet3guac_core_config.json index c14e6af1f0..eff5d1e24c 100644 --- a/environment_configs/sre_bluet3guac_core_config.json +++ b/environment_configs/sre_bluet3guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.240"], diff --git a/environment_configs/sre_bluet3msrds_core_config.json b/environment_configs/sre_bluet3msrds_core_config.json index c83c0833ab..56381578db 100644 --- a/environment_configs/sre_bluet3msrds_core_config.json +++ b/environment_configs/sre_bluet3msrds_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "MicrosoftRDS", "dataAdminIpAddresses": ["193.60.220.240"], diff --git a/environment_configs/sre_greent1guac_core_config.json b/environment_configs/sre_greent1guac_core_config.json index e78af97072..10cad425c3 100644 --- a/environment_configs/sre_greent1guac_core_config.json +++ b/environment_configs/sre_greent1guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent2guac_core_config.json b/environment_configs/sre_greent2guac_core_config.json index f0e726431c..091a74e605 100644 --- a/environment_configs/sre_greent2guac_core_config.json +++ b/environment_configs/sre_greent2guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent3msrds_core_config.json b/environment_configs/sre_greent3msrds_core_config.json index 1cd114f919..118e24da48 100644 --- a/environment_configs/sre_greent3msrds_core_config.json +++ b/environment_configs/sre_greent3msrds_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "MicrosoftRDS", "dataAdminIpAddresses": ["193.60.220.240"], From b0a95b982d8cddc213125069a9a5ac3503869a4d Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 1 Sep 2023 13:58:37 +0000 Subject: [PATCH 31/33] Update pester test config files --- tests/resources/shm_blue_full_config.json | 7 +++ tests/resources/shm_green_full_config.json | 7 +++ .../resources/sre_bluet1guac_full_config.json | 45 +++++++++++-------- .../sre_bluet3msrds_full_config.json | 45 +++++++++++-------- .../sre_greent2guac_full_config.json | 45 +++++++++++-------- 5 files changed, 92 insertions(+), 57 deletions(-) diff --git a/tests/resources/shm_blue_full_config.json b/tests/resources/shm_blue_full_config.json index ddc78b8f59..7172383965 100644 --- a/tests/resources/shm_blue_full_config.json +++ b/tests/resources/shm_blue_full_config.json @@ -150,6 +150,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", diff --git a/tests/resources/shm_green_full_config.json b/tests/resources/shm_green_full_config.json index d47cd3edb8..65ba0efadf 100644 --- a/tests/resources/shm_green_full_config.json +++ b/tests/resources/shm_green_full_config.json @@ -150,6 +150,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", diff --git a/tests/resources/sre_bluet1guac_full_config.json b/tests/resources/sre_bluet1guac_full_config.json index 24f1f18747..61e843d6b0 100644 --- a/tests/resources/sre_bluet1guac_full_config.json +++ b/tests/resources/sre_bluet1guac_full_config.json @@ -151,6 +151,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", @@ -1111,15 +1118,8 @@ "instances": [ { "adminPasswordSecretName": "sre-t1guac-vm-admin-password-mssql", - "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql", "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-mssql", - "vmName": "MSSQL-T1GUAC", - "type": "MSSQL", - "ip": "10.151.3.4", - "port": "1433", - "sku": "sqldev-gen2", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql", "disks": { "data": { "sizeGb": "1024", @@ -1130,19 +1130,19 @@ "type": "Standard_LRS" } }, - "enableSSIS": true + "enableSSIS": true, + "ip": "10.151.3.4", + "port": "1433", + "sku": "sqldev-gen2", + "subnet": "databases", + "type": "MSSQL", + "vmName": "MSSQL-T1GUAC", + "vmSize": "Standard_DS2_v2" }, { "adminPasswordSecretName": "sre-t1guac-vm-admin-password-postgresql", - "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql", "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-postgresql", - "vmName": "PSTGRS-T1GUAC", - "type": "PostgreSQL", - "ip": "10.151.3.5", - "port": "5432", - "sku": "Ubuntu-latest", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql", "disks": { "data": { "sizeGb": "1024", @@ -1152,7 +1152,14 @@ "sizeGb": "128", "type": "Standard_LRS" } - } + }, + "ip": "10.151.3.5", + "port": "5432", + "sku": "Ubuntu-latest", + "subnet": "databases", + "type": "PostgreSQL", + "vmName": "PSTGRS-T1GUAC", + "vmSize": "Standard_DS2_v2" } ], "rg": "RG_SHM_BLUE_SRE_T1GUAC_DATABASES" @@ -1302,7 +1309,7 @@ "rg": "RG_SHM_BLUE_SRE_T1GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "vmSizeDefault": "Standard_D2s_v3" }, diff --git a/tests/resources/sre_bluet3msrds_full_config.json b/tests/resources/sre_bluet3msrds_full_config.json index 1a59d3b537..44c1673fb3 100644 --- a/tests/resources/sre_bluet3msrds_full_config.json +++ b/tests/resources/sre_bluet3msrds_full_config.json @@ -151,6 +151,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", @@ -1111,15 +1118,8 @@ "instances": [ { "adminPasswordSecretName": "sre-t3msrds-vm-admin-password-mssql", - "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-mssql", "dbAdminPasswordSecretName": "sre-t3msrds-db-admin-password-mssql", - "vmName": "MSSQL-T3MSRDS", - "type": "MSSQL", - "ip": "10.163.3.4", - "port": "1433", - "sku": "sqldev-gen2", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-mssql", "disks": { "data": { "sizeGb": "1024", @@ -1130,19 +1130,19 @@ "type": "Standard_LRS" } }, - "enableSSIS": true + "enableSSIS": true, + "ip": "10.163.3.4", + "port": "1433", + "sku": "sqldev-gen2", + "subnet": "databases", + "type": "MSSQL", + "vmName": "MSSQL-T3MSRDS", + "vmSize": "Standard_DS2_v2" }, { "adminPasswordSecretName": "sre-t3msrds-vm-admin-password-postgresql", - "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-postgresql", "dbAdminPasswordSecretName": "sre-t3msrds-db-admin-password-postgresql", - "vmName": "PSTGRS-T3MSRDS", - "type": "PostgreSQL", - "ip": "10.163.3.5", - "port": "5432", - "sku": "Ubuntu-latest", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t3msrds-db-admin-username-postgresql", "disks": { "data": { "sizeGb": "1024", @@ -1152,7 +1152,14 @@ "sizeGb": "128", "type": "Standard_LRS" } - } + }, + "ip": "10.163.3.5", + "port": "5432", + "sku": "Ubuntu-latest", + "subnet": "databases", + "type": "PostgreSQL", + "vmName": "PSTGRS-T3MSRDS", + "vmSize": "Standard_DS2_v2" } ], "rg": "RG_SHM_BLUE_SRE_T3MSRDS_DATABASES" @@ -1324,7 +1331,7 @@ "rg": "RG_SHM_BLUE_SRE_T3MSRDS_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "vmSizeDefault": "Standard_D2s_v3" }, diff --git a/tests/resources/sre_greent2guac_full_config.json b/tests/resources/sre_greent2guac_full_config.json index ca318c0553..73ca82eae5 100644 --- a/tests/resources/sre_greent2guac_full_config.json +++ b/tests/resources/sre_greent2guac_full_config.json @@ -151,6 +151,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", @@ -1164,15 +1171,8 @@ "instances": [ { "adminPasswordSecretName": "sre-t2guac-vm-admin-password-mssql", - "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-mssql", "dbAdminPasswordSecretName": "sre-t2guac-db-admin-password-mssql", - "vmName": "MSSQL-T2GUAC", - "type": "MSSQL", - "ip": "10.152.3.4", - "port": "1433", - "sku": "sqldev-gen2", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-mssql", "disks": { "data": { "sizeGb": "1024", @@ -1183,19 +1183,19 @@ "type": "Standard_LRS" } }, - "enableSSIS": true + "enableSSIS": true, + "ip": "10.152.3.4", + "port": "1433", + "sku": "sqldev-gen2", + "subnet": "databases", + "type": "MSSQL", + "vmName": "MSSQL-T2GUAC", + "vmSize": "Standard_DS2_v2" }, { "adminPasswordSecretName": "sre-t2guac-vm-admin-password-postgresql", - "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-postgresql", "dbAdminPasswordSecretName": "sre-t2guac-db-admin-password-postgresql", - "vmName": "PSTGRS-T2GUAC", - "type": "PostgreSQL", - "ip": "10.152.3.5", - "port": "5432", - "sku": "Ubuntu-latest", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-postgresql", "disks": { "data": { "sizeGb": "1024", @@ -1205,7 +1205,14 @@ "sizeGb": "128", "type": "Standard_LRS" } - } + }, + "ip": "10.152.3.5", + "port": "5432", + "sku": "Ubuntu-latest", + "subnet": "databases", + "type": "PostgreSQL", + "vmName": "PSTGRS-T2GUAC", + "vmSize": "Standard_DS2_v2" } ], "rg": "RG_SHM_GREEN_SRE_T2GUAC_DATABASES" @@ -1355,7 +1362,7 @@ "rg": "RG_SHM_GREEN_SRE_T2GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "vmSizeDefault": "Standard_D2s_v3" }, From 656bd1420d47d160996205de73b7880bc6573840 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:30:19 +0000 Subject: [PATCH 32/33] Finalise merge docs from latest into develop --- docs/source/deployment/deploy_sre.md | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/source/deployment/deploy_sre.md b/docs/source/deployment/deploy_sre.md index e5c5353b42..fe95d4b391 100644 --- a/docs/source/deployment/deploy_sre.md +++ b/docs/source/deployment/deploy_sre.md @@ -105,6 +105,41 @@ PS> ./Setup_SRE_Guacamole_Servers.ps1 -shmId -sreId
+
+Update SSL certificate + +![Powershell: five minutes](https://img.shields.io/static/v1?style=for-the-badge&logo=powershell&label=local&color=blue&message=five%20minutes) at {{file_folder}} `./deployment/secure_research_environment/setup` + +```powershell +PS> ./Update_SRE_SSL_Certificate.ps1 -shmId -sreId +``` + +- where `` is the {ref}`management environment ID ` for this SHM +- where `` is the {ref}`secure research environment ID ` for this SRE +- where `` is an email address that you want to be notified when certificates are close to expiry + +```{tip} +`./Update_SRE_RDS_SSL_Certificate.ps1` should be run again whenever you want to update the certificate for this SRE. +``` + +```{caution} +`Let's Encrypt` will only issue **5 certificates per week** for a particular host (e.g. `rdg-sre-sandbox.project.turingsafehaven.ac.uk`). +To reduce the number of calls to `Let's Encrypt`, the signed certificates are stored in the Key Vault for easy redeployment. +For production environments this should usually not be an issue. +``` + +````{important} +If you find yourself frequently redeploying a test environment and hit the `Let's Encrypt` certificate limit, you can can use: + +```powershell +> ./Update_SRE_RDS_SSL_Certificate.ps1 -dryRun $true +``` + +to use the `Let's Encrypt` staging server, which will issue certificates more frequently. +These certificates will **not** be trusted by your browser, and so should not be used in production. +```` +
+
Deploy web applications (CodiMD and GitLab) From 15936985e2b4a613534f06e1f67809dabd78b67a Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:35:36 +0000 Subject: [PATCH 33/33] Fix linting error --- docs/source/deployment/deploy_sre.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/deployment/deploy_sre.md b/docs/source/deployment/deploy_sre.md index fe95d4b391..74f909bb48 100644 --- a/docs/source/deployment/deploy_sre.md +++ b/docs/source/deployment/deploy_sre.md @@ -138,6 +138,7 @@ If you find yourself frequently redeploying a test environment and hit the `Let' to use the `Let's Encrypt` staging server, which will issue certificates more frequently. These certificates will **not** be trusted by your browser, and so should not be used in production. ```` +