From ef2c23619de9d318cd54ce82b4ac3e7b485174f3 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:52:37 +0000 Subject: [PATCH 01/78] Use sphinxcontrib-typer and install all dependencies for docs environment --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b12c64a3ed..3abf5ad8d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,7 @@ docs = [ "pydata-sphinx-theme==0.15.4", "sphinx-togglebutton==0.3.2", "sphinx==8.1.3", + "sphinxcontrib-typer==0.5.1", ] lint = [ "ansible-dev-tools==24.9.0", @@ -120,7 +121,6 @@ lock-filename = ".hatch/requirements.txt" [tool.hatch.envs.docs] type = "pip-compile" lock-filename = ".hatch/requirements-docs.txt" -detached = true features = ["docs"] [tool.hatch.envs.docs.scripts] From c9dda0e9e3b8bd0cc9a6f60fa3289df80b542dd7 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:54:16 +0000 Subject: [PATCH 02/78] use sphinxcontrib-typer extension and configure sphinx appropriately --- docs/source/conf.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index f262d36dc2..f2d17d4d84 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -5,7 +5,9 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html import datetime import emoji - +import sys +import os +sys.path.insert(0, os.path.abspath(os.path.join("..", ".."))) # -- Project information ----------------------------------------------------- @@ -55,6 +57,7 @@ extensions = [ "myst_parser", "sphinx_togglebutton", + "sphinxcontrib.typer", ] # Add any paths that contain templates here, relative to this directory. From 6e778adb61a54044955febf7cce8b0b4873f3f8e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:10:11 +0000 Subject: [PATCH 03/78] Add command reference top-level section --- docs/source/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/index.md b/docs/source/index.md index 764b06ec38..64588350d7 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -9,6 +9,7 @@ design/index.md deployment/index.md management/index.md roles/index.md +reference/index.md contributing/index.md ::: @@ -42,6 +43,8 @@ You can read them through in order or simply jump to the section that you are mo - If you want to deploy your own Data Safe Haven. - [**Management**](management/index.md) - If you want to manage a Data Safe Haven that you (or someone else) has deployed. +- [**Reference**](reference/index.md) + - If you want details of all the Command Line Interface - [**Roles**](roles/index.md) - Information about the different user roles in the Data Safe Haven. - Instructions and advice for the actions of different user roles. From 7f3b96e822c748254ce68031cb62e24d004e579b Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:13:47 +0000 Subject: [PATCH 04/78] add all commands to reference --- docs/source/reference/index.md | 64 ++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 docs/source/reference/index.md diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md new file mode 100644 index 0000000000..88c5434d57 --- /dev/null +++ b/docs/source/reference/index.md @@ -0,0 +1,64 @@ +# Reference + +## Main command + +:::{typer} data_safe_haven.commands.cli:application +:preferred: text +:prog: dsh +:width: 65 +::: + +## Config commands + +:::{typer} data_safe_haven.commands.config:config_command_group +:width: 65 +:prog: dsh config +:show-nested: +:make-sections: +::: + +## Context commands + +:::{typer} data_safe_haven.commands.context:context_command_group +:width: 65 +:prog: dsh context +:show-nested: +:make-sections: +::: + +## SHM commands + +:::{typer} data_safe_haven.commands.shm:shm_command_group +:width: 65 +:prog: dsh shm +:show-nested: +:make-sections: +::: + +## SRE commands + +:::{typer} data_safe_haven.commands.sre:sre_command_group +:width: 65 +:prog: dsh sre +:show-nested: +:make-sections: +::: + +## User commands + +:::{typer} data_safe_haven.commands.users:users_command_group +:width: 65 +:prog: dsh users +:show-nested: +:make-sections: +::: + + +## Pulumi commands + +:::{typer} data_safe_haven.commands.pulumi:pulumi_command_group +:width: 65 +:prog: dsh pulumi +:show-nested: +:make-sections: +::: \ No newline at end of file From 1827b4d5119dbd240c333ab956e5470a5d4e857e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:14:01 +0000 Subject: [PATCH 05/78] Add blank line --- docs/source/reference/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index 88c5434d57..635a4c4cbd 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -61,4 +61,4 @@ :prog: dsh pulumi :show-nested: :make-sections: -::: \ No newline at end of file +::: From db05b5cab2a5e76f47ad64700435071dc4e1f249 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:23:01 +0000 Subject: [PATCH 06/78] use consistent format for main dsh command --- docs/source/reference/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index 635a4c4cbd..c15efe8128 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -3,7 +3,6 @@ ## Main command :::{typer} data_safe_haven.commands.cli:application -:preferred: text :prog: dsh :width: 65 ::: From 15d30c5f88eb221e457f0c14338c3d3d4f173343 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:37:26 +0000 Subject: [PATCH 07/78] use better title --- docs/source/reference/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index c15efe8128..b469875780 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -1,6 +1,6 @@ # Reference -## Main command +## `dsh` command line interface :::{typer} data_safe_haven.commands.cli:application :prog: dsh From 68562c62edbf13d34383ceca7ed9650e16690934 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:37:41 +0000 Subject: [PATCH 08/78] update requirements for docs --- .hatch/requirements-docs.txt | 284 ++++++++++++++++++++++++++++++++++- 1 file changed, 280 insertions(+), 4 deletions(-) diff --git a/.hatch/requirements-docs.txt b/.hatch/requirements-docs.txt index e4ac178af1..beaf72b641 100644 --- a/.hatch/requirements-docs.txt +++ b/.hatch/requirements-docs.txt @@ -1,17 +1,125 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.12 # +# - appdirs==1.4.4 +# - azure-core==1.31.0 +# - azure-identity==1.19.0 +# - azure-keyvault-certificates==4.8.0 +# - azure-keyvault-keys==4.9.0 +# - azure-keyvault-secrets==4.8.0 +# - azure-mgmt-compute==33.0.0 +# - azure-mgmt-containerinstance==10.1.0 +# - azure-mgmt-dns==8.1.0 +# - azure-mgmt-keyvault==10.3.1 +# - azure-mgmt-msi==7.0.0 +# - azure-mgmt-rdbms==10.1.0 +# - azure-mgmt-resource==23.1.1 +# - azure-mgmt-storage==21.2.1 +# - azure-storage-blob==12.23.1 +# - azure-storage-file-datalake==12.17.0 +# - azure-storage-file-share==12.19.0 +# - chevron==0.14.0 +# - cryptography==43.0.1 +# - fqdn==1.5.1 +# - psycopg[binary]==3.2.3 +# - pulumi-azure-native==2.66.0 +# - pulumi-azuread==6.0.0 +# - pulumi-random==4.16.6 +# - pulumi==3.136.1 +# - pydantic==2.9.2 +# - pyjwt[crypto]==2.9.0 +# - pytz==2024.2 +# - pyyaml==6.0.2 +# - rich==13.9.2 +# - simple-acme-dns==3.1.0 +# - typer==0.12.5 +# - websocket-client==1.8.0 # - emoji==2.14.0 # - myst-parser==4.0.0 # - pydata-sphinx-theme==0.15.4 # - sphinx-togglebutton==0.3.2 # - sphinx==8.1.3 +# - sphinxcontrib-typer==0.5.1 # accessible-pygments==0.0.5 # via pydata-sphinx-theme +acme==2.10.0 + # via simple-acme-dns alabaster==1.0.0 # via sphinx +annotated-types==0.7.0 + # via pydantic +appdirs==1.4.4 + # via hatch.envs.docs +arpeggio==2.0.2 + # via parver +attrs==24.2.0 + # via parver +azure-common==1.1.28 + # via + # azure-mgmt-compute + # azure-mgmt-containerinstance + # azure-mgmt-dns + # azure-mgmt-keyvault + # azure-mgmt-msi + # azure-mgmt-rdbms + # azure-mgmt-resource + # azure-mgmt-storage +azure-core==1.31.0 + # via + # hatch.envs.docs + # azure-identity + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-mgmt-core + # azure-storage-blob + # azure-storage-file-datalake + # azure-storage-file-share + # msrest +azure-identity==1.19.0 + # via hatch.envs.docs +azure-keyvault-certificates==4.8.0 + # via hatch.envs.docs +azure-keyvault-keys==4.9.0 + # via hatch.envs.docs +azure-keyvault-secrets==4.8.0 + # via hatch.envs.docs +azure-mgmt-compute==33.0.0 + # via hatch.envs.docs +azure-mgmt-containerinstance==10.1.0 + # via hatch.envs.docs +azure-mgmt-core==1.4.0 + # via + # azure-mgmt-compute + # azure-mgmt-containerinstance + # azure-mgmt-dns + # azure-mgmt-keyvault + # azure-mgmt-msi + # azure-mgmt-rdbms + # azure-mgmt-resource + # azure-mgmt-storage +azure-mgmt-dns==8.1.0 + # via hatch.envs.docs +azure-mgmt-keyvault==10.3.1 + # via hatch.envs.docs +azure-mgmt-msi==7.0.0 + # via hatch.envs.docs +azure-mgmt-rdbms==10.1.0 + # via hatch.envs.docs +azure-mgmt-resource==23.1.1 + # via hatch.envs.docs +azure-mgmt-storage==21.2.1 + # via hatch.envs.docs +azure-storage-blob==12.23.1 + # via + # hatch.envs.docs + # azure-storage-file-datalake +azure-storage-file-datalake==12.17.0 + # via hatch.envs.docs +azure-storage-file-share==12.19.0 + # via hatch.envs.docs babel==2.16.0 # via # pydata-sphinx-theme @@ -19,9 +127,37 @@ babel==2.16.0 beautifulsoup4==4.12.3 # via pydata-sphinx-theme certifi==2024.8.30 - # via requests + # via + # msrest + # requests +cffi==1.17.1 + # via cryptography charset-normalizer==3.4.0 # via requests +chevron==0.14.0 + # via hatch.envs.docs +click==8.1.7 + # via + # typer + # typer-slim +cryptography==43.0.1 + # via + # hatch.envs.docs + # acme + # azure-identity + # azure-keyvault-keys + # azure-storage-blob + # azure-storage-file-share + # josepy + # msal + # pyjwt + # pyopenssl +debugpy==1.8.7 + # via pulumi +dill==0.3.9 + # via pulumi +dnspython==2.6.1 + # via simple-acme-dns docutils==0.21.2 # via # myst-parser @@ -30,41 +166,154 @@ docutils==0.21.2 # sphinx-togglebutton emoji==2.14.0 # via hatch.envs.docs +fqdn==1.5.1 + # via hatch.envs.docs +grpcio==1.66.2 + # via pulumi idna==3.10 # via requests imagesize==1.4.1 # via sphinx +isodate==0.7.2 + # via + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-mgmt-compute + # azure-mgmt-containerinstance + # azure-mgmt-dns + # azure-mgmt-keyvault + # azure-mgmt-resource + # azure-mgmt-storage + # azure-storage-blob + # azure-storage-file-datalake + # azure-storage-file-share + # msrest jinja2==3.1.4 # via # myst-parser # sphinx +josepy==1.14.0 + # via acme markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser + # rich markupsafe==3.0.1 # via jinja2 mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py +msal==1.31.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.2.0 + # via azure-identity +msrest==0.7.1 + # via + # azure-mgmt-msi + # azure-mgmt-rdbms myst-parser==4.0.0 # via hatch.envs.docs +oauthlib==3.2.2 + # via requests-oauthlib packaging==24.1 # via # pydata-sphinx-theme # sphinx +parver==0.5 + # via + # pulumi-azure-native + # pulumi-azuread + # pulumi-random +portalocker==2.10.1 + # via msal-extensions +protobuf==4.25.5 + # via pulumi +psycopg==3.2.3 + # via hatch.envs.docs +psycopg-binary==3.2.3 + # via psycopg +pulumi==3.136.1 + # via + # hatch.envs.docs + # pulumi-azure-native + # pulumi-azuread + # pulumi-random +pulumi-azure-native==2.66.0 + # via hatch.envs.docs +pulumi-azuread==6.0.0 + # via hatch.envs.docs +pulumi-random==4.16.6 + # via hatch.envs.docs +pycparser==2.22 + # via cffi +pydantic==2.9.2 + # via hatch.envs.docs +pydantic-core==2.23.4 + # via pydantic pydata-sphinx-theme==0.15.4 # via hatch.envs.docs pygments==2.18.0 # via # accessible-pygments # pydata-sphinx-theme + # rich # sphinx +pyjwt==2.9.0 + # via + # hatch.envs.docs + # msal +pyopenssl==24.2.1 + # via + # acme + # josepy +pyrfc3339==1.1 + # via acme +pytz==2024.2 + # via + # hatch.envs.docs + # acme + # pyrfc3339 pyyaml==6.0.2 - # via myst-parser + # via + # hatch.envs.docs + # myst-parser + # pulumi requests==2.32.3 - # via sphinx + # via + # acme + # azure-core + # msal + # msrest + # requests-oauthlib + # sphinx +requests-oauthlib==2.0.0 + # via msrest +rich==13.9.2 + # via + # hatch.envs.docs + # typer + # typer-slim +semver==2.13.0 + # via + # pulumi + # pulumi-azure-native + # pulumi-azuread + # pulumi-random +shellingham==1.5.4 + # via + # typer + # typer-slim +simple-acme-dns==3.1.0 + # via hatch.envs.docs +six==1.16.0 + # via + # azure-core + # pulumi snowballstemmer==2.2.0 # via sphinx soupsieve==2.6 @@ -75,6 +324,7 @@ sphinx==8.1.3 # myst-parser # pydata-sphinx-theme # sphinx-togglebutton + # sphinxcontrib-typer sphinx-togglebutton==0.3.2 # via hatch.envs.docs sphinxcontrib-applehelp==2.0.0 @@ -89,10 +339,36 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx +sphinxcontrib-typer==0.5.1 + # via hatch.envs.docs +typer==0.12.5 + # via hatch.envs.docs +typer-slim==0.12.5 + # via sphinxcontrib-typer typing-extensions==4.12.2 - # via pydata-sphinx-theme + # via + # azure-core + # azure-identity + # azure-keyvault-certificates + # azure-keyvault-keys + # azure-keyvault-secrets + # azure-mgmt-compute + # azure-mgmt-keyvault + # azure-storage-blob + # azure-storage-file-datalake + # azure-storage-file-share + # psycopg + # pydantic + # pydantic-core + # pydata-sphinx-theme + # typer + # typer-slim urllib3==2.2.3 # via requests +validators==0.28.3 + # via simple-acme-dns +websocket-client==1.8.0 + # via hatch.envs.docs wheel==0.44.0 # via sphinx-togglebutton From 1a34235692ef06eed02d8395cbc0bc94d47c0c7a Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:40:44 +0000 Subject: [PATCH 09/78] Fix linting --- docs/source/reference/index.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index b469875780..e76787573c 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -52,7 +52,6 @@ :make-sections: ::: - ## Pulumi commands :::{typer} data_safe_haven.commands.pulumi:pulumi_command_group From 95619ecfbe459b04ecf82214f49a7b5a4592c2de Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:38:33 +0000 Subject: [PATCH 10/78] Check that a user belongs to the correct SHM domain when registering with SRE --- data_safe_haven/commands/users.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index fe413fa781..d90783f39b 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -146,11 +146,22 @@ def register( # List users users = UserHandler(context, graph_api) - available_usernames = users.get_usernames_entra_id() + # available_usernames = users.get_usernames_entra_id() + available_users = users.entra_users.list() + user_dict = {user.user_principal_name.split('@')[0]: user.user_principal_name.split('@')[1] for user in available_users} usernames_to_register = [] for username in usernames: - if username in available_usernames: - usernames_to_register.append(username) + if username in user_dict.keys(): + user_domain = user_dict[username] + if user_domain != shm_config.shm.fqdn: + logger.error( + f"Username '{username}' belongs to SHM domain '{user_domain}'.\n" + f"SRE '{sre_config.name}' is associated with SHM domain '{shm_config.shm.fqdn}'.\n" + "Users can only be registered to one SHM domain.\n" + "Please use 'dsh users add' to create a new user associated with the current SHM domain." + ) + else: + usernames_to_register.append(username) else: logger.error( f"Username '{username}' does not belong to this Data Safe Haven deployment." From 2943ab281959ce534a8f6171bfc23eb5d668b998 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:03:19 +0000 Subject: [PATCH 11/78] fix linting --- data_safe_haven/commands/users.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index d90783f39b..6c28096ded 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -148,7 +148,12 @@ def register( users = UserHandler(context, graph_api) # available_usernames = users.get_usernames_entra_id() available_users = users.entra_users.list() - user_dict = {user.user_principal_name.split('@')[0]: user.user_principal_name.split('@')[1] for user in available_users} + user_dict = { + user.user_principal_name.split("@")[0]: user.user_principal_name.split("@")[ + 1 + ] + for user in available_users + } usernames_to_register = [] for username in usernames: if username in user_dict.keys(): From fdcce90075480cc3835b8f44a542c3e5a1485550 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:48:06 +0000 Subject: [PATCH 12/78] get fqdn from stack output --- data_safe_haven/commands/users.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 6c28096ded..d479a1393e 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -10,6 +10,7 @@ from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi from data_safe_haven.logging import get_logger +from data_safe_haven.infrastructure import SREProjectManager users_command_group = typer.Typer() @@ -121,8 +122,9 @@ def register( try: shm_config = SHMConfig.from_remote(context) except DataSafeHavenError: - logger.error("Have you deployed the SHM?") - raise + msg = "Have you deployed the SHM?" + logger.error(msg) + raise DataSafeHavenError(msg) # Load Pulumi config pulumi_config = DSHPulumiConfig.from_remote(context) @@ -132,7 +134,13 @@ def register( if sre_config.name not in pulumi_config.project_names: msg = f"Could not load Pulumi settings for '{sre_config.name}'. Have you deployed the SRE?" logger.error(msg) - raise DataSafeHavenError(msg) + raise typer.Exit(1) + + sre_stack = SREProjectManager( + context=context, + config=sre_config, + pulumi_config=pulumi_config, + ) # Load GraphAPI graph_api = GraphApi.from_scopes( @@ -155,13 +163,14 @@ def register( for user in available_users } usernames_to_register = [] + shm_name = sre_stack.output("linked_shm")["name"] for username in usernames: if username in user_dict.keys(): user_domain = user_dict[username] - if user_domain != shm_config.shm.fqdn: + if shm_name not in user_domain: logger.error( f"Username '{username}' belongs to SHM domain '{user_domain}'.\n" - f"SRE '{sre_config.name}' is associated with SHM domain '{shm_config.shm.fqdn}'.\n" + f"SRE '{sre_config.name}' is associated with SHM domain '{shm_name}'.\n" "Users can only be registered to one SHM domain.\n" "Please use 'dsh users add' to create a new user associated with the current SHM domain." ) From 8b89c9c376e3f2c2a9df645e81c5f83a64bf16f6 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:49:34 +0000 Subject: [PATCH 13/78] Add exports from SRE stack --- data_safe_haven/infrastructure/programs/declarative_sre.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 15989bbe7b..9b0a5952ba 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -420,4 +420,6 @@ def __call__(self) -> None: pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) pulumi.export("remote_desktop", remote_desktop.exports) + pulumi.export("sre_fqdn", networking.sre_fqdn) + pulumi.export("linked_shm", self.context.name) pulumi.export("workspaces", workspaces.exports) From ca7483d87a3c468dc40b25d4ab5efa081a754edf Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:20:16 +0000 Subject: [PATCH 14/78] Update release checklist Remove irrelevant names section, add release making steps --- .github/ISSUE_TEMPLATE/release_checklist.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index a25064faa1..ade631052e 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -44,16 +44,16 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo - [ ] Update supported versions in `SECURITY.md` - [ ] Update pen test results in `VERSIONING.md` -## :computer: Release information +### Making the release -- **Version number:** _ -- **SHM ID:** _ -- **T2 SRE ID:** _ -- **T3 SRE ID:** _ +- [ ] Merge release branch into `latest` +- [ ] Push tag in the format `v0.0.1` to the merge commit into `latest` +- [ ] Ensure docs for the latest version are built and deployed on ReadTheDocs +- [ ] Push a build to PyPI +- [ ] Announce release on communications channels ## :deciduous_tree: Deployment problems - From 147939cacab07de554612a30c0c317593050e351 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:21:53 +0000 Subject: [PATCH 15/78] Remove powershell from bug report template --- .github/ISSUE_TEMPLATE/deployment_bug_report.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/deployment_bug_report.md b/.github/ISSUE_TEMPLATE/deployment_bug_report.md index 6cf453cc13..fa569038c2 100644 --- a/.github/ISSUE_TEMPLATE/deployment_bug_report.md +++ b/.github/ISSUE_TEMPLATE/deployment_bug_report.md @@ -29,7 +29,6 @@ Before reporting a problem please check the following. Replace the empty checkbo List of packages From 8a9df7a166a21f41ec6aa66858375db29b9392b0 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:22:52 +0000 Subject: [PATCH 16/78] Remove unused scripts --- .github/scripts/update_azure_data_studio.py | 19 ------ .github/scripts/update_dbeaver_drivers.py | 72 --------------------- .github/scripts/update_rstudio.py | 21 ------ 3 files changed, 112 deletions(-) delete mode 100644 .github/scripts/update_azure_data_studio.py delete mode 100644 .github/scripts/update_dbeaver_drivers.py delete mode 100644 .github/scripts/update_rstudio.py diff --git a/.github/scripts/update_azure_data_studio.py b/.github/scripts/update_azure_data_studio.py deleted file mode 100644 index 651e85fdfc..0000000000 --- a/.github/scripts/update_azure_data_studio.py +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env python3 -from lxml import html -import hashlib -import requests - -remote_page = requests.get("https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio", allow_redirects=True) -root = html.fromstring(remote_page.content) -short_link = root.xpath("//a[contains(text(), '.deb')]/@href")[0] - -remote_content = requests.get(short_link, allow_redirects=True) -sha256 = hashlib.sha256(remote_content.content).hexdigest() -version = remote_content.url.split("-")[-1].replace(".deb", "") -remote = "/".join(remote_content.url.split("/")[:-1] + ["|DEBFILE|"]) - -with open("deployment/secure_research_desktop/packages/deb-azuredatastudio.version", "w") as f_out: - f_out.write(f"hash: {sha256}\n") - f_out.write(f"version: {version}\n") - f_out.write("debfile: azuredatastudio-linux-|VERSION|.deb\n") - f_out.write(f"remote: {remote}\n") diff --git a/.github/scripts/update_dbeaver_drivers.py b/.github/scripts/update_dbeaver_drivers.py deleted file mode 100644 index 696a501858..0000000000 --- a/.github/scripts/update_dbeaver_drivers.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python3 -import json -from lxml import html -from natsort import natsorted -import requests - - -def get_latest_version(url, search_text): - """ - Get latest version number of a database driver from the Maven repository. - - Fetches the HTML page at the given URL, then converts it to an lxml tree. - Numeric strings are then extracted. - Note that mostly numeric strings for some drivers contain non-numeric text, - as different driver types exist for those drivers, even where the version number is the same. - The largest (latest) version number of the driver is then returned. - - Parameters - ---------- - url : str - The URL of the Maven repository containing the driver - search_text : str - Text to search for in the repository, to distinguish the driver from other files - - Returns - ------- - list - The latest available version number of the driver - """ - - remote_page = requests.get(url, allow_redirects=True) - root = html.fromstring(remote_page.content) - return natsorted([v for v in root.xpath("//a[contains(text(), '" + search_text + "')]/@href") if v != "../"])[-1].replace("/", "") - - -drivers = [ - { - 'name': "mssql_jdbc", - 'url': "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", - 'search_text': "jre8/" - }, - { - 'name': "pgjdbc", - 'url': "https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", - 'search_text': "/" - }, - { - 'name': "postgresql", - 'url': "https://repo1.maven.org/maven2/org/postgresql/postgresql/", - 'search_text': "/" - }, - { - 'name': "postgis_geometry", - 'url': "https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", - 'search_text': "/" - }, - { - 'name': "postgis_jdbc", - 'url': "https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", - 'search_text': "/" - }, - { - 'name': "waffle_jna", - 'url': "https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/", - 'search_text': "/" - } -] - -output = {driver['name']: get_latest_version(driver['url'], driver['search_text']) for driver in drivers} - -with open("deployment/secure_research_desktop/packages/dbeaver-driver-versions.json", "w") as f_out: - f_out.writelines(json.dumps(output, indent=4, sort_keys=True)) diff --git a/.github/scripts/update_rstudio.py b/.github/scripts/update_rstudio.py deleted file mode 100644 index ee36a35e66..0000000000 --- a/.github/scripts/update_rstudio.py +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/env python3 -from lxml import html -import hashlib -import requests - -remote_page = requests.get("https://www.rstudio.com/products/rstudio/download/", allow_redirects=True) -root = html.fromstring(remote_page.content) -short_links = [link for link in root.xpath("//a[contains(text(), '.deb')]/@href") if "debian" not in link] - -for ubuntu_version in ["focal", "jammy"]: - short_link = [link for link in short_links if ubuntu_version in link][0] - remote_content = requests.get(short_link, allow_redirects=True) - sha256 = hashlib.sha256(remote_content.content).hexdigest() - version = "-".join(remote_content.url.split("/")[-1].split("-")[1:-1]) - remote = "/".join(remote_content.url.split("/")[:-1] + ["|DEBFILE|"]) - - with open(f"deployment/secure_research_desktop/packages/deb-rstudio-{ubuntu_version}.version", "w") as f_out: - f_out.write(f"hash: {sha256}\n") - f_out.write(f"version: {version}\n") - f_out.write("debfile: rstudio-|VERSION|-amd64.deb\n") - f_out.write(f"remote: {remote}\n") From d4aed3047975b0cb097528274948ea805d7b8cf9 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:23:04 +0000 Subject: [PATCH 17/78] Remove security checklist template --- .github/security_checklist_template.md | 167 ------------------------- 1 file changed, 167 deletions(-) delete mode 100644 .github/security_checklist_template.md diff --git a/.github/security_checklist_template.md b/.github/security_checklist_template.md deleted file mode 100644 index b963331eef..0000000000 --- a/.github/security_checklist_template.md +++ /dev/null @@ -1,167 +0,0 @@ -# Security checklist -Running on SHM/SREs deployed using commit XXXXXXX - -## Summary -+ :white_check_mark: N tests passed -- :partly_sunny: N tests partially passed (see below for more details) -- :fast_forward: N tests skipped (see below for more details) -- :x: N tests failed (see below for more details) - -## Details -Some security checks were skipped since: -- No managed device was available -- No access to a physical space with its own dedicated network was possible - -### Multifactor Authentication and Password strength -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the SRE standard user cannot access the apps - +
:camera: Verify before adding to group: Microsoft Remote Desktop: Login works but apps cannot be viewed - -
- +
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA - -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that adding the **SRE standard user** to the SRE group on the domain controller does not give them access - +
:camera: Verify after adding to group: Microsoft Remote Desktop: Login works and apps can be viewed - -
- +
:camera: Verify after adding to group: Microsoft Remote Desktop: attempt to login to DSVM Main (Desktop) fails - -
- +
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA - -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** is able to successfully set up MFA - +
:camera: Verify: successfully set up MFA - -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can authenticate with MFA - +
:camera: Verify: Guacamole: respond to the MFA prompt - 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> -
- +
:camera: Verify: Microsoft Remote Desktop: attempt to log in to DSVM Main (Desktop) and respond to the MFA prompt - 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can access the DSVM desktop - +
:camera: Verify: Microsoft Remote Desktop: connect to DSVM Main (Desktop) - -
- +
:camera: Verify: Guacamole: connect to Desktop: Ubuntu0 - -
- -### Isolated Network -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connect to the SHM DC and NPS if connected to the SHM VPN -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the SHM DC and NPS if not connected to the SHM VPN -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from within a DSVM on the SRE network. - +
:camera: Verify: Connection fails - 122045859-8142bb00-cdde-11eb-920c-3a162a180647.png"> -
- +
:camera: Verify: that you cannot access a website using curl - -
- +
:camera: Verify: that you cannot get the IP address for a website using nslookup - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that users cannot connect between two SREs within the same SHM, even if they have access to both SREs - +
:camera: Verify: SSH connection fails - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules are set appropriately to block outgoing traffic - +
:camera: Verify: access rules - -
- -### User devices -#### Tier 2: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection - -#### Tier 3: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check user lacks root access -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection with an allow-listed IP address - -#### Tiers 2+: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses - +
:camera: Verify: access rules - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: All non-deployment NSGs have rules denying inbound connections from outside the Virtual Network - -### Physical security -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from outside was not tested -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from inside was not tested -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check the network IP ranges corresponding to the research spaces and compare against the IPs accepted by the firewall. -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so confirmation of physical measures was not tested - -### Remote connections - -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH - +
:camera: Verify: SSH connection by FQDN fails - -
- +
:camera: Verify: SSH connection by public IP address fails - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: The remote desktop server is the only SRE resource with a public IP address - -### Copy-and-paste -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to paste local text into a DSVM -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to copy text from a DSVM -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Copy between VMs in an SRE succeeds - -### Data ingress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** secure upload token successfully created with write-only permissions -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** token was sent using a secure, out-of-band communication channel (e.g. secure email) -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an allow-listed IP address succeeds -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** downloading a file from an allow-listed IP address fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an non-allowed IP address fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection during lifetime of short-duration token succeeds -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection after lifetime of short-duration token fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading different file types succeeds - -### Storage volumes and egress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/output` volume -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can only read from the `/data` volume -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to their directory in `/home` -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/shared` volume -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can see the files ready for egress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can download egress-ready files - -### Software Ingress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** expected software tools are installed - +
:camera: Verify: DBeaver, RStudio, PyCharm and Visual Studio Code available - 122056611-0a132400-cdea-11eb-9087-385ab296189e.png"> -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** secure upload token successfully created with write-only permissions -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading is possible only during the token lifetime -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** uploaded files are readable and can be installed on the DSVM -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** uploaded files are readable but cannot be installed on the DSVM - -### Package mirrors - -#### Tier 2: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages - +
:camera: Verify: botocore can be installed - -
- -#### Tier 3: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages - +
:camera: Verify: aero-calc can be installed; botocore cannot be installed - -
- -### Azure firewalls -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Admin has limited access to the internet - +
:camera: Verify: SHM DC cannot connect to google - 122067607-ff5d8c80-cdf3-11eb-8e20-a401faba0be4.png"> -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Admin can download Windows updates - +
:camera: Verify: Windows updates can be downloaded - 122067641-071d3100-cdf4-11eb-9dc8-03938ff49e3a.png"> -
From 408b2d02fbc1a3c23c45295b0b479e0e2b449a45 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 25 Nov 2024 15:23:40 +0000 Subject: [PATCH 18/78] Add diagnostic settings for firewall --- .../programs/declarative_sre.py | 29 +++++++-------- .../infrastructure/programs/sre/firewall.py | 35 ++++++++++++++++++- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 78467f201b..f69fc9cd45 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -163,12 +163,27 @@ def __call__(self) -> None: ), ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy SRE firewall SREFirewallComponent( "sre_firewall", self.stack_name, SREFirewallProps( location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group_name=resource_group.name, route_table_name=networking.route_table_name, subnet_apt_proxy_server=networking.subnet_apt_proxy_server, @@ -209,20 +224,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy the apt proxy server apt_proxy_server = SREAptProxyServerComponent( "sre_apt_proxy_server", diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 97f7a885b7..1f46db980b 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -3,12 +3,13 @@ from collections.abc import Mapping from pulumi import ComponentResource, Input, Output, ResourceOptions -from pulumi_azure_native import network +from pulumi_azure_native import insights, network from data_safe_haven.infrastructure.common import ( get_address_prefixes_from_subnet, get_id_from_subnet, ) +from data_safe_haven.infrastructure.components import WrappedLogAnalyticsWorkspace from data_safe_haven.types import ( FirewallPriorities, ForbiddenDomains, @@ -23,6 +24,7 @@ class SREFirewallProps: def __init__( self, location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], route_table_name: Input[str], subnet_apt_proxy_server: Input[network.GetSubnetResult], @@ -35,6 +37,7 @@ def __init__( subnet_workspaces: Input[network.GetSubnetResult], ) -> None: self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.route_table_name = route_table_name self.subnet_apt_proxy_server_prefixes = Output.from_input( @@ -331,6 +334,36 @@ def __init__( tags=child_tags, ) + # Add diagnostic settings for firewall + # This links the firewall to the log analytics workspace + insights.DiagnosticSettings( + f"{self._name}_firewall_diagnostic_settings", + name="firewall_diagnostic_settings", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "AllMetrics", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=firewall.id, + workspace_id=props.log_analytics_workspace.workspace_id, + ) + # Retrieve the private IP address for the firewall private_ip_address = firewall.ip_configurations.apply( lambda cfgs: "" if not cfgs else cfgs[0].private_ip_address From a09802a36b77c29f8198dc8ca6a5dd08cea0ca77 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 25 Nov 2024 16:33:29 +0000 Subject: [PATCH 19/78] Correct component name --- data_safe_haven/infrastructure/programs/sre/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 1f46db980b..4e45aff208 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -336,7 +336,7 @@ def __init__( # Add diagnostic settings for firewall # This links the firewall to the log analytics workspace - insights.DiagnosticSettings( + insights.DiagnosticSetting( f"{self._name}_firewall_diagnostic_settings", name="firewall_diagnostic_settings", log_analytics_destination_type="Dedicated", From d886b102c5c01f3b0dc8e5ce8f575cf50b19b2be Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:10:07 +0000 Subject: [PATCH 20/78] Clarify error message and check against SRE context --- data_safe_haven/commands/users.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index d479a1393e..01dd79e6c8 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -9,8 +9,8 @@ from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi -from data_safe_haven.logging import get_logger from data_safe_haven.infrastructure import SREProjectManager +from data_safe_haven.logging import get_logger users_command_group = typer.Typer() @@ -122,9 +122,8 @@ def register( try: shm_config = SHMConfig.from_remote(context) except DataSafeHavenError: - msg = "Have you deployed the SHM?" - logger.error(msg) - raise DataSafeHavenError(msg) + logger.error("Have you deployed the SHM?") + raise typer.Exit(1) # Load Pulumi config pulumi_config = DSHPulumiConfig.from_remote(context) @@ -154,7 +153,6 @@ def register( # List users users = UserHandler(context, graph_api) - # available_usernames = users.get_usernames_entra_id() available_users = users.entra_users.list() user_dict = { user.user_principal_name.split("@")[0]: user.user_principal_name.split("@")[ @@ -163,16 +161,15 @@ def register( for user in available_users } usernames_to_register = [] - shm_name = sre_stack.output("linked_shm")["name"] + shm_name = sre_stack.output("context") for username in usernames: if username in user_dict.keys(): - user_domain = user_dict[username] + user_domain = user_dict[username].split(".")[0] if shm_name not in user_domain: logger.error( - f"Username '{username}' belongs to SHM domain '{user_domain}'.\n" - f"SRE '{sre_config.name}' is associated with SHM domain '{shm_name}'.\n" - "Users can only be registered to one SHM domain.\n" - "Please use 'dsh users add' to create a new user associated with the current SHM domain." + f"Username [green]'{username}'[/green] belongs to SHM context [blue]'{user_domain}'[/blue].\n" + f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM context [blue]'{shm_name}'[/blue].\n" + "The user must belong to the same SHM context as the SRE." ) else: usernames_to_register.append(username) From 01585a1b0c1a283fd594803873be81dead49913b Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:10:54 +0000 Subject: [PATCH 21/78] Fix linting --- data_safe_haven/commands/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 01dd79e6c8..b3456c8784 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -121,9 +121,9 @@ def register( # Load SHMConfig try: shm_config = SHMConfig.from_remote(context) - except DataSafeHavenError: + except DataSafeHavenError as exc: logger.error("Have you deployed the SHM?") - raise typer.Exit(1) + raise typer.Exit(1) from exc # Load Pulumi config pulumi_config = DSHPulumiConfig.from_remote(context) From d5042b2cd262630f4c6a644c1707fd238e178436 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:11:17 +0000 Subject: [PATCH 22/78] Export context name associated with SRE as part of SRE stack --- data_safe_haven/infrastructure/programs/declarative_sre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 6ce9c81d1d..d9d6f0b545 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -424,7 +424,7 @@ def __call__(self) -> None: # Export values for later use pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) + pulumi.export("context", self.context.name) pulumi.export("remote_desktop", remote_desktop.exports) pulumi.export("sre_fqdn", networking.sre_fqdn) - pulumi.export("linked_shm", self.context.name) pulumi.export("workspaces", workspaces.exports) From 134404b068255d281d6567c349267361b15d6104 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:41:00 +0000 Subject: [PATCH 23/78] Correct list indent --- .github/ISSUE_TEMPLATE/release_checklist.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index ade631052e..8686c95238 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -34,10 +34,10 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo ### For major releases only - [ ] Confirm that a third party has carried out a full penetration test evaluating: - 1. external attack surface - 1. ability to exfiltrate data from the system - 1. ability to transfer data between SREs - 1. ability to escalate privileges on the SRD. + 1. external attack surface + 1. ability to exfiltrate data from the system + 1. ability to transfer data between SREs + 1. ability to escalate privileges on the SRD. ### Update documentation From bb0eaf2c391781bb0219ad76b427a235162f98f8 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:48:49 +0000 Subject: [PATCH 24/78] Remove bare URL --- .github/ISSUE_TEMPLATE/release_checklist.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 8686c95238..42acf2fe51 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -14,9 +14,9 @@ Before reporting a problem please check the following. Replace the empty checkbo Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deployment) section of our documentation when completing these steps. -- [ ] Consult the `data-safe-haven/VERSIONING.md` guide and determine the version number of the new release. Record it in the title of this issue. +- [ ] Consult the `data-safe-haven/VERSIONING.md` guide and determine the version number of the new release. Record it in the title of this issue - [ ] Create a release branch called e.g. `release-v0.0.1` -- [ ] Draft a changelog for the release similar to our previous releases, see https://github.com/alan-turing-institute/data-safe-haven/releases +- [ ] Draft a changelog for the release similar to our [previous releases](https://github.com/alan-turing-institute/data-safe-haven/releases) ### For patch releases only From 56081d87bff505215239d3f19f8f46a0483ea28c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:49:04 +0000 Subject: [PATCH 25/78] Clarify tag and release creation --- .github/ISSUE_TEMPLATE/release_checklist.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 42acf2fe51..575f5c9c53 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -47,7 +47,8 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo ### Making the release - [ ] Merge release branch into `latest` -- [ ] Push tag in the format `v0.0.1` to the merge commit into `latest` +- [ ] Create a tag of the form `v0.0.1` pointing to the most recent commit on `latest` (the merge that you just made) +- [ ] Publish your draft GitHub release using this tag - [ ] Ensure docs for the latest version are built and deployed on ReadTheDocs - [ ] Push a build to PyPI - [ ] Announce release on communications channels From d020b3e7ed764a42167e80adba37b1d854119a70 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:58:48 +0000 Subject: [PATCH 26/78] Use full resource URI --- data_safe_haven/infrastructure/programs/sre/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 4e45aff208..ed831e826a 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -361,7 +361,7 @@ def __init__( } ], resource_uri=firewall.id, - workspace_id=props.log_analytics_workspace.workspace_id, + workspace_id=props.log_analytics_workspace.id, ) # Retrieve the private IP address for the firewall From 757dcf1a3a7f0ea292aa9adb8e694cc8246287ff Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:11:03 +0000 Subject: [PATCH 27/78] Use SHM FQDN in error message rather than context name, clarify error message --- data_safe_haven/commands/users.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index b3456c8784..2db3cf5bd3 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -164,19 +164,19 @@ def register( shm_name = sre_stack.output("context") for username in usernames: if username in user_dict.keys(): - user_domain = user_dict[username].split(".")[0] + user_domain = user_dict[username] if shm_name not in user_domain: logger.error( - f"Username [green]'{username}'[/green] belongs to SHM context [blue]'{user_domain}'[/blue].\n" - f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM context [blue]'{shm_name}'[/blue].\n" - "The user must belong to the same SHM context as the SRE." + f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" + f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" + "The user's principal domain name must match the domain of the SRE to be registered." ) else: usernames_to_register.append(username) else: logger.error( f"Username '{username}' does not belong to this Data Safe Haven deployment." - " Please use 'dsh users add' to create it." + "Please use 'dsh users add' to create this user." ) users.register(sre_config.name, usernames_to_register) except DataSafeHavenError as exc: From 510a7149983c2cb97f9bd0d5a7d3d4b04a4c7fdb Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:15:46 +0000 Subject: [PATCH 28/78] Remove unneeded export of context name --- data_safe_haven/commands/users.py | 3 +-- data_safe_haven/infrastructure/programs/declarative_sre.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 2db3cf5bd3..e9e96a5283 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -161,11 +161,10 @@ def register( for user in available_users } usernames_to_register = [] - shm_name = sre_stack.output("context") for username in usernames: if username in user_dict.keys(): user_domain = user_dict[username] - if shm_name not in user_domain: + if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index d9d6f0b545..78467f201b 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -424,7 +424,6 @@ def __call__(self) -> None: # Export values for later use pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) - pulumi.export("context", self.context.name) pulumi.export("remote_desktop", remote_desktop.exports) pulumi.export("sre_fqdn", networking.sre_fqdn) pulumi.export("workspaces", workspaces.exports) From 309d777298356b8f1070909dd4409c76ee3233b8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:29:38 +0000 Subject: [PATCH 29/78] Remove unnecessary loading of SRE stack --- data_safe_haven/commands/users.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index e9e96a5283..5dab2e7591 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -9,7 +9,6 @@ from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi -from data_safe_haven.infrastructure import SREProjectManager from data_safe_haven.logging import get_logger users_command_group = typer.Typer() @@ -135,12 +134,6 @@ def register( logger.error(msg) raise typer.Exit(1) - sre_stack = SREProjectManager( - context=context, - config=sre_config, - pulumi_config=pulumi_config, - ) - # Load GraphAPI graph_api = GraphApi.from_scopes( scopes=["Group.ReadWrite.All", "GroupMember.ReadWrite.All"], From fa08bc3a5d324d231b7e7b759ed53117a220410f Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 10:58:31 +0000 Subject: [PATCH 30/78] Add documentation for firewall logs --- docs/source/management/logs.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index f9a9948453..5d52fe5d77 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -21,7 +21,7 @@ These include, - Gitea and Hedgedoc Logs from all containers are ingested into the [SREs log workspace](#log-workspace). -There are two logs +There are two tables, `ContainerEvents_CL` : Event logs for the container instance resources such as starting, stopping, crashes and pulling images. @@ -29,3 +29,22 @@ There are two logs `ContainerInstanceLog_CL` : Container process logs. : This is where you can view the output of the containerised applications and will be useful for debugging problems. + +## Firewall logs + +The firewall plays a critical role in the security of a Data Safe Haven. +It filters all outbound traffic through a set of FQDN rules so that each component may only reach necessary and allowed domains. + +Logs from the firewall are ingested into the [SREs log workspace](#log-workspace). +There are multiple tables, + +`AZFWApplicationRule` +: Logs from the firewalls FDQN filters. +: Shows requests to the outside of the Data Safe Haven and why they have been approved or rejected. + +`AZFWDnsQuery` +: DNS requests handled by the firewall. + +`AzureMetrics` +: Various metrics on firewall utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. From fd6dfec1f5330fb2b014dd79d92cf09f66e3ace0 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 11:57:32 +0000 Subject: [PATCH 31/78] Add diagnostic setting for NFSv3 containers --- .../composite/nfsv3_blob_container.py | 44 ++++++++++++++++++- .../infrastructure/programs/sre/data.py | 5 +++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index 98564918a0..f4275fed8e 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,9 +1,10 @@ from pulumi import ComponentResource, Input, ResourceOptions -from pulumi_azure_native import storage +from pulumi_azure_native import insights, storage -from data_safe_haven.infrastructure.components.dynamic.blob_container_acl import ( +from data_safe_haven.infrastructure.components import ( BlobContainerAcl, BlobContainerAclProps, + WrappedLogAnalyticsWorkspace, ) @@ -15,6 +16,7 @@ def __init__( acl_other: Input[str], apply_default_permissions: Input[bool], container_name: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account: Input[storage.StorageAccount], subscription_name: Input[str], @@ -24,6 +26,7 @@ def __init__( self.acl_other = acl_other self.apply_default_permissions = apply_default_permissions self.container_name = container_name + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account = storage_account self.subscription_name = subscription_name @@ -52,6 +55,7 @@ def __init__( ResourceOptions(parent=props.storage_account), ), ) + BlobContainerAcl( f"{storage_container._name}_acl", BlobContainerAclProps( @@ -70,6 +74,42 @@ def __init__( ), ) + insights.DiagnosticSetting( + f"{storage_container._name}_diagnostic_settings", + name="firewall_diagnostic_settings", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_container.id, + workspace_id=props.log_analytics_workspace.id, + ) + self.name = storage_container.name self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 711b76139f..f478b56df8 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -33,6 +33,7 @@ NFSV3BlobContainerProps, SSLCertificate, SSLCertificateProps, + WrappedLogAnalyticsWorkspace, WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -51,6 +52,7 @@ def __init__( dns_record: Input[network.RecordSet], dns_server_admin_password: Input[pulumi_random.RandomPassword], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], storage_quota_gb_home: Input[int], @@ -69,6 +71,7 @@ def __init__( self.dns_record = dns_record self.password_dns_server_admin = dns_server_admin_password self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -492,6 +495,7 @@ def __init__( # 65533 ownership of the fileshare (preventing use inside the SRE) apply_default_permissions=False, container_name="egress", + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, @@ -507,6 +511,7 @@ def __init__( # files (eg. with Azure Storage Explorer) apply_default_permissions=True, container_name="ingress", + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, From bd045eaf8c33fdcf86905906cb3a31a2f49a5328 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:52:18 +0000 Subject: [PATCH 32/78] user preferred_name instead of directly accessing principal name --- data_safe_haven/commands/users.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 5dab2e7591..53473f58ed 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -148,9 +148,7 @@ def register( users = UserHandler(context, graph_api) available_users = users.entra_users.list() user_dict = { - user.user_principal_name.split("@")[0]: user.user_principal_name.split("@")[ - 1 - ] + user.preferred_username.split("@")[0]: user.preferred_username.split("@")[1] for user in available_users } usernames_to_register = [] From 6dc5a688bf0efcf5f1ec1c053c14f26d78b3441e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 14:27:24 +0000 Subject: [PATCH 33/78] Correct imports --- .../components/composite/nfsv3_blob_container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index f4275fed8e..96d6e8a8b0 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,9 +1,11 @@ from pulumi import ComponentResource, Input, ResourceOptions from pulumi_azure_native import insights, storage -from data_safe_haven.infrastructure.components import ( +from data_safe_haven.infrastructure.components.dynamic import ( BlobContainerAcl, BlobContainerAclProps, +) +from data_safe_haven.infrastructure.components.wrapped import ( WrappedLogAnalyticsWorkspace, ) From 9bc497f23b05d9a7ebd665e8f3d73280fde07ed5 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 14:36:10 +0000 Subject: [PATCH 34/78] Pass log workspace to data component --- .../programs/declarative_sre.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 78467f201b..96df9380b2 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -183,6 +183,20 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy data storage data = SREDataComponent( "sre_data", @@ -196,6 +210,7 @@ def __call__(self) -> None: dns_record=networking.shm_ns_record, dns_server_admin_password=dns.password_admin, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, storage_quota_gb_home=self.config.sre.storage_quota_gb.home, @@ -209,20 +224,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy the apt proxy server apt_proxy_server = SREAptProxyServerComponent( "sre_apt_proxy_server", From 78754ae57421113b1c8f95d85926c64380e3dd69 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 15:35:53 +0000 Subject: [PATCH 35/78] Move diagnostic setting to storage account --- .../composite/nfsv3_blob_container.py | 43 +------------------ .../infrastructure/programs/sre/data.py | 41 +++++++++++++++++- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index 96d6e8a8b0..29550e9541 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,13 +1,10 @@ from pulumi import ComponentResource, Input, ResourceOptions -from pulumi_azure_native import insights, storage +from pulumi_azure_native import storage from data_safe_haven.infrastructure.components.dynamic import ( BlobContainerAcl, BlobContainerAclProps, ) -from data_safe_haven.infrastructure.components.wrapped import ( - WrappedLogAnalyticsWorkspace, -) class NFSV3BlobContainerProps: @@ -18,7 +15,6 @@ def __init__( acl_other: Input[str], apply_default_permissions: Input[bool], container_name: Input[str], - log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account: Input[storage.StorageAccount], subscription_name: Input[str], @@ -28,7 +24,6 @@ def __init__( self.acl_other = acl_other self.apply_default_permissions = apply_default_permissions self.container_name = container_name - self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account = storage_account self.subscription_name = subscription_name @@ -76,42 +71,6 @@ def __init__( ), ) - insights.DiagnosticSetting( - f"{storage_container._name}_diagnostic_settings", - name="firewall_diagnostic_settings", - log_analytics_destination_type="Dedicated", - logs=[ - { - "category_group": "allLogs", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - { - "category_group": "audit", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - ], - metrics=[ - { - "category": "Transaction", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - } - ], - resource_uri=storage_container.id, - workspace_id=props.log_analytics_workspace.id, - ) - self.name = storage_container.name self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index f478b56df8..fcca697810 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,6 +7,7 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, + insights, keyvault, managedidentity, network, @@ -495,7 +496,6 @@ def __init__( # 65533 ownership of the fileshare (preventing use inside the SRE) apply_default_permissions=False, container_name="egress", - log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, @@ -511,12 +511,49 @@ def __init__( # files (eg. with Azure Storage Explorer) apply_default_permissions=True, container_name="ingress", - log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, ), ) + # Add diagnostic setting for blobs + insights.DiagnosticSetting( + f"{storage_account_data_private_sensitive._name}_diagnostic_setting", + name=f"{storage_account_data_private_sensitive._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_private_sensitive.id.apply( + lambda resource_id: resource_id + "/blobServices" + ), + workspace_id=props.log_analytics_workspace.id, + ) # Set up a private endpoint for the sensitive data storage account storage_account_data_private_sensitive_endpoint = network.PrivateEndpoint( f"{storage_account_data_private_sensitive._name}_private_endpoint", From 469f1d6ac57bb81e8e4bebb7b4a84d897c7af163 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:40:04 +0000 Subject: [PATCH 36/78] better formatted error message --- data_safe_haven/commands/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 53473f58ed..b3a90c0664 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -158,14 +158,14 @@ def register( if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" - f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" + f"SRE [yellow]'{sre}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" "The user's principal domain name must match the domain of the SRE to be registered." ) else: usernames_to_register.append(username) else: logger.error( - f"Username '{username}' does not belong to this Data Safe Haven deployment." + f"Username '{username}' does not belong to this Data Safe Haven deployment.\n" "Please use 'dsh users add' to create this user." ) users.register(sre_config.name, usernames_to_register) From 2abd14b27984ac2df81b3da60ffdf8d537927019 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:40:52 +0000 Subject: [PATCH 37/78] add test for mismatched domain --- tests/commands/conftest.py | 13 +++++++++++++ tests/commands/test_users.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index d675398bfc..de60eb29d0 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -1,6 +1,8 @@ from pytest import fixture from typer.testing import CliRunner +from data_safe_haven.administration.users.entra_users import EntraUsers +from data_safe_haven.administration.users.research_user import ResearchUser from data_safe_haven.config import ( Context, ContextManager, @@ -260,3 +262,14 @@ def tmp_contexts_none(tmp_path, context_yaml): with open(config_file_path, "w") as f: f.write(context_yaml) return tmp_path + + +@fixture +def mock_entra_user_list(mocker): + test_user = ResearchUser( + given_name="Harry", + surname="Lime", + sam_account_name="harry.lime", + user_principal_name="harry.lime@acme.testing", + ) + mocker.patch.object(EntraUsers, "list", return_value=[test_user]) diff --git a/tests/commands/test_users.py b/tests/commands/test_users.py index c1b183c922..5c11e29cc9 100644 --- a/tests/commands/test_users.py +++ b/tests/commands/test_users.py @@ -52,6 +52,26 @@ def test_invalid_shm( assert result.exit_code == 1 assert "Have you deployed the SHM?" in result.stdout + def test_mismatched_domain( + self, + mock_graphapi_get_credential, # noqa: ARG002 + mock_pulumi_config_no_key_from_remote, # noqa: ARG002 + mock_shm_config_from_remote, # noqa: ARG002 + mock_sre_config_from_remote, # noqa: ARG002 + mock_entra_user_list, # noqa: ARG002 + runner, + tmp_contexts, # noqa: ARG002 + ): + result = runner.invoke( + users_command_group, ["register", "-u", "harry.lime", "sandbox"] + ) + + assert result.exit_code == 0 + assert ( + "principal domain name must match the domain of the SRE to be registered" + in result.stdout + ) + def test_invalid_sre( self, mock_pulumi_config_from_remote, # noqa: ARG002 From 4519383d4abf5e4986ad25bac20923a1990cc0d9 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:28:57 +0000 Subject: [PATCH 38/78] update requirements-docs --- .hatch/requirements-docs.txt | 110 ++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 35 deletions(-) diff --git a/.hatch/requirements-docs.txt b/.hatch/requirements-docs.txt index 1a8db2f195..2a11901466 100644 --- a/.hatch/requirements-docs.txt +++ b/.hatch/requirements-docs.txt @@ -2,37 +2,37 @@ # This file is autogenerated by hatch-pip-compile with Python 3.12 # # - appdirs==1.4.4 -# - azure-core==1.31.0 +# - azure-core==1.32.0 # - azure-identity==1.19.0 -# - azure-keyvault-certificates==4.8.0 -# - azure-keyvault-keys==4.9.0 -# - azure-keyvault-secrets==4.8.0 +# - azure-keyvault-certificates==4.9.0 +# - azure-keyvault-keys==4.10.0 +# - azure-keyvault-secrets==4.9.0 # - azure-mgmt-compute==33.0.0 # - azure-mgmt-containerinstance==10.1.0 -# - azure-mgmt-dns==8.1.0 +# - azure-mgmt-dns==8.2.0 # - azure-mgmt-keyvault==10.3.1 # - azure-mgmt-msi==7.0.0 # - azure-mgmt-rdbms==10.1.0 -# - azure-mgmt-resource==23.1.1 +# - azure-mgmt-resource==23.2.0 # - azure-mgmt-storage==21.2.1 -# - azure-storage-blob==12.23.1 -# - azure-storage-file-datalake==12.17.0 -# - azure-storage-file-share==12.19.0 +# - azure-storage-blob==12.24.0 +# - azure-storage-file-datalake==12.18.0 +# - azure-storage-file-share==12.20.0 # - chevron==0.14.0 -# - cryptography==43.0.1 +# - cryptography==43.0.3 # - fqdn==1.5.1 -# - psycopg[binary]==3.2.3 -# - pulumi-azure-native==2.66.0 -# - pulumi-azuread==6.0.0 -# - pulumi-random==4.16.6 -# - pulumi==3.136.1 -# - pydantic==2.9.2 -# - pyjwt[crypto]==2.9.0 +# - psycopg[binary]==3.1.19 +# - pulumi-azure-native==2.73.1 +# - pulumi-azuread==6.0.1 +# - pulumi-random==4.16.7 +# - pulumi==3.141.0 +# - pydantic==2.10.1 +# - pyjwt[crypto]==2.10.0 # - pytz==2024.2 # - pyyaml==6.0.2 -# - rich==13.9.2 -# - simple-acme-dns==3.1.0 -# - typer==0.12.5 +# - rich==13.9.4 +# - simple-acme-dns==3.2.0 +# - typer==0.13.1 # - websocket-client==1.8.0 # - emoji==2.14.0 # - myst-parser==4.0.0 @@ -66,7 +66,7 @@ azure-common==1.1.28 # azure-mgmt-rdbms # azure-mgmt-resource # azure-mgmt-storage -azure-core==1.31.0 +azure-core==1.32.0 # via # hatch.envs.docs # azure-identity @@ -80,11 +80,11 @@ azure-core==1.31.0 # msrest azure-identity==1.19.0 # via hatch.envs.docs -azure-keyvault-certificates==4.8.0 +azure-keyvault-certificates==4.9.0 # via hatch.envs.docs -azure-keyvault-keys==4.9.0 +azure-keyvault-keys==4.10.0 # via hatch.envs.docs -azure-keyvault-secrets==4.8.0 +azure-keyvault-secrets==4.9.0 # via hatch.envs.docs azure-mgmt-compute==33.0.0 # via hatch.envs.docs @@ -100,7 +100,7 @@ azure-mgmt-core==1.4.0 # azure-mgmt-rdbms # azure-mgmt-resource # azure-mgmt-storage -azure-mgmt-dns==8.1.0 +azure-mgmt-dns==8.2.0 # via hatch.envs.docs azure-mgmt-keyvault==10.3.1 # via hatch.envs.docs @@ -108,17 +108,17 @@ azure-mgmt-msi==7.0.0 # via hatch.envs.docs azure-mgmt-rdbms==10.1.0 # via hatch.envs.docs -azure-mgmt-resource==23.1.1 +azure-mgmt-resource==23.2.0 # via hatch.envs.docs azure-mgmt-storage==21.2.1 # via hatch.envs.docs -azure-storage-blob==12.23.1 +azure-storage-blob==12.24.0 # via # hatch.envs.docs # azure-storage-file-datalake -azure-storage-file-datalake==12.17.0 +azure-storage-file-datalake==12.18.0 # via hatch.envs.docs -azure-storage-file-share==12.19.0 +azure-storage-file-share==12.20.0 # via hatch.envs.docs babel==2.16.0 # via @@ -140,7 +140,7 @@ click==8.1.7 # via # typer # typer-slim -cryptography==43.0.1 +cryptography==43.0.3 # via # hatch.envs.docs # acme @@ -156,7 +156,7 @@ debugpy==1.8.7 # via pulumi dill==0.3.9 # via pulumi -dnspython==2.6.1 +dnspython==2.7.0 # via simple-acme-dns docutils==0.21.2 # via @@ -199,6 +199,7 @@ markdown-it-py==3.0.0 # via # mdit-py-plugins # myst-parser + # rich markupsafe==3.0.2 # via jinja2 mdit-py-plugins==0.4.2 @@ -217,8 +218,41 @@ msrest==0.7.1 # azure-mgmt-rdbms myst-parser==4.0.0 # via hatch.envs.docs +oauthlib==3.2.2 + # via requests-oauthlib packaging==24.2 # via sphinx +parver==0.5 + # via + # pulumi-azure-native + # pulumi-azuread + # pulumi-random +portalocker==2.10.1 + # via msal-extensions +protobuf==4.25.5 + # via pulumi +psycopg==3.1.19 + # via hatch.envs.docs +psycopg-binary==3.1.19 + # via psycopg +pulumi==3.141.0 + # via + # hatch.envs.docs + # pulumi-azure-native + # pulumi-azuread + # pulumi-random +pulumi-azure-native==2.73.1 + # via hatch.envs.docs +pulumi-azuread==6.0.1 + # via hatch.envs.docs +pulumi-random==4.16.7 + # via hatch.envs.docs +pycparser==2.22 + # via cffi +pydantic==2.10.1 + # via hatch.envs.docs +pydantic-core==2.27.1 + # via pydantic pydata-sphinx-theme==0.16.0 # via hatch.envs.docs pygments==2.18.0 @@ -227,7 +261,7 @@ pygments==2.18.0 # pydata-sphinx-theme # rich # sphinx -pyjwt==2.9.0 +pyjwt==2.10.0 # via # hatch.envs.docs # msal @@ -257,7 +291,7 @@ requests==2.32.3 # sphinx requests-oauthlib==2.0.0 # via msrest -rich==13.9.2 +rich==13.9.4 # via # hatch.envs.docs # typer @@ -272,7 +306,7 @@ shellingham==1.5.4 # via # typer # typer-slim -simple-acme-dns==3.1.0 +simple-acme-dns==3.2.0 # via hatch.envs.docs six==1.16.0 # via @@ -305,7 +339,7 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sphinxcontrib-typer==0.5.1 # via hatch.envs.docs -typer==0.12.5 +typer==0.13.1 # via hatch.envs.docs typer-slim==0.12.5 # via sphinxcontrib-typer @@ -317,7 +351,9 @@ typing-extensions==4.12.2 # azure-keyvault-keys # azure-keyvault-secrets # azure-mgmt-compute + # azure-mgmt-dns # azure-mgmt-keyvault + # azure-mgmt-resource # azure-storage-blob # azure-storage-file-datalake # azure-storage-file-share @@ -329,6 +365,10 @@ typing-extensions==4.12.2 # typer-slim urllib3==2.2.3 # via requests +validators==0.34.0 + # via simple-acme-dns +websocket-client==1.8.0 + # via hatch.envs.docs wheel==0.45.1 # via sphinx-togglebutton From 23926e5d3c4151c50d4b218844db493cd41f3e9d Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:12:11 +0000 Subject: [PATCH 39/78] Add workspace log docs --- docs/source/management/logs.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index f9a9948453..0b575fde2d 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -20,7 +20,7 @@ These include, - package proxy - Gitea and Hedgedoc -Logs from all containers are ingested into the [SREs log workspace](#log-workspace). +Logs from all containers are ingested into the [SRE's log analytics workspace](#log-workspace). There are two logs `ContainerEvents_CL` @@ -29,3 +29,18 @@ There are two logs `ContainerInstanceLog_CL` : Container process logs. : This is where you can view the output of the containerised applications and will be useful for debugging problems. + +## Workspace logs + +Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace). + +There are three logs + +`Perf` +: Usage statistics for individual workspaces, such as percent memory used and percent disk space used + +`Syslog` +: Linux system logs for individual workspaces, useful for debugging problems related to system processes + +`Heartbeat` +: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) \ No newline at end of file From 52f587ac824cee29652d63b8686f066272c7e8ca Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:13:12 +0000 Subject: [PATCH 40/78] fix linting --- docs/source/management/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 0b575fde2d..3c5b9c7b18 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -43,4 +43,4 @@ There are three logs : Linux system logs for individual workspaces, useful for debugging problems related to system processes `Heartbeat` -: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) \ No newline at end of file +: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) From dbcd646ce830ab31da58144d63bc682d03427c99 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:14:40 +0000 Subject: [PATCH 41/78] Add link to docs for Azure Monitor Agent --- docs/source/management/logs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 3c5b9c7b18..046f9c12eb 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -7,7 +7,7 @@ Some of these logs are ingested into a central location, an Azure [Log Analytics Each SRE has its own Log Analytics Workspace. You can view the workspaces by going to the Azure portal and navigating to [Log Analytics Workspaces](https://portal.azure.com/#browse/Microsoft.OperationalInsights%2Fworkspaces). -Select which log workspace you want to view by clicking on the workspace named `shm--sre--log`. +Select which Log Analytics Workspace you want to view by clicking on the workspace named `shm--sre--log`. The logs can be filtered using [Kusto Query Language (KQL)](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview). @@ -32,7 +32,7 @@ There are two logs ## Workspace logs -Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace). +Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace) using the [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/azure-monitor-agent-overview). There are three logs @@ -43,4 +43,4 @@ There are three logs : Linux system logs for individual workspaces, useful for debugging problems related to system processes `Heartbeat` -: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) +: Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) From 9f0b6a6b876ac7b7a5fd428f7f993f80178e322b Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:15:37 +0000 Subject: [PATCH 42/78] Add some brief text at the start of each section --- docs/source/reference/index.md | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index e76787573c..93faa71c80 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -2,12 +2,17 @@ ## `dsh` command line interface +The `dsh` commands are the entrypoint to the Data Safe Haven command line interface. +All commands begin with `dsh`. + :::{typer} data_safe_haven.commands.cli:application :prog: dsh :width: 65 ::: -## Config commands +## `config` commands + +`config` commands are used to manage the configuration files that define SHMs and SREs. :::{typer} data_safe_haven.commands.config:config_command_group :width: 65 @@ -16,7 +21,9 @@ :make-sections: ::: -## Context commands +## `context` commands + +`context` commands are used to manage the Data Safe Haven contexts, which are the grouping within which a single SHM and its associated SREs are organised. :::{typer} data_safe_haven.commands.context:context_command_group :width: 65 @@ -25,7 +32,9 @@ :make-sections: ::: -## SHM commands +## `shm` commands + +`shm` commands are used to deploy or teardown DSH Safe Haven Management infrastructure :::{typer} data_safe_haven.commands.shm:shm_command_group :width: 65 @@ -34,7 +43,9 @@ :make-sections: ::: -## SRE commands +## `sre` commands + +`sre` commands are used to deploy or teardown the infrastructure for DSH Secure Research Environments :::{typer} data_safe_haven.commands.sre:sre_command_group :width: 65 @@ -43,7 +54,9 @@ :make-sections: ::: -## User commands +## `users` commands + +`users` commands are used to manage users on the Entra ID associated with a DSH deployment. :::{typer} data_safe_haven.commands.users:users_command_group :width: 65 @@ -52,7 +65,7 @@ :make-sections: ::: -## Pulumi commands +## `pulumi` commands :::{typer} data_safe_haven.commands.pulumi:pulumi_command_group :width: 65 From 4792c11d456eeb09163805cf24139f06bfa27c99 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:32:57 +0000 Subject: [PATCH 43/78] Break reference into individual pages --- docs/source/reference/config.md | 10 ++++ docs/source/reference/context.md | 10 ++++ docs/source/reference/index.md | 85 ++++++++++---------------------- docs/source/reference/pulumi.md | 8 +++ docs/source/reference/shm.md | 10 ++++ docs/source/reference/sre.md | 10 ++++ docs/source/reference/users.md | 10 ++++ 7 files changed, 85 insertions(+), 58 deletions(-) create mode 100644 docs/source/reference/config.md create mode 100644 docs/source/reference/context.md create mode 100644 docs/source/reference/pulumi.md create mode 100644 docs/source/reference/shm.md create mode 100644 docs/source/reference/sre.md create mode 100644 docs/source/reference/users.md diff --git a/docs/source/reference/config.md b/docs/source/reference/config.md new file mode 100644 index 0000000000..daffc4e2e8 --- /dev/null +++ b/docs/source/reference/config.md @@ -0,0 +1,10 @@ +# `config` + +`dsh config` commands are used to manage the configuration files that define SHMs and SREs. + +:::{typer} data_safe_haven.commands.config:config_command_group +:width: 65 +:prog: dsh config +:show-nested: +:make-sections: +::: diff --git a/docs/source/reference/context.md b/docs/source/reference/context.md new file mode 100644 index 0000000000..76b72a4ca0 --- /dev/null +++ b/docs/source/reference/context.md @@ -0,0 +1,10 @@ +# `context` + +`dsh context` commands are used to manage the Data Safe Haven contexts, which are the grouping within which a single SHM and its associated SREs are organised. + +:::{typer} data_safe_haven.commands.context:context_command_group +:width: 65 +:prog: dsh context +:show-nested: +:make-sections: +::: diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index 93faa71c80..d790c7d448 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -1,6 +1,18 @@ # Reference -## `dsh` command line interface +:::{toctree} +:hidden: + +config.md +context.md +users.md +pulumi.md +shm.md +sre.md +::: + +A Data Safe Haven is managed using the `dsh` command line interface. +A full guide to the commands available for managing your Data Safe Haven is provided here. The `dsh` commands are the entrypoint to the Data Safe Haven command line interface. All commands begin with `dsh`. @@ -10,66 +22,23 @@ All commands begin with `dsh`. :width: 65 ::: -## `config` commands - -`config` commands are used to manage the configuration files that define SHMs and SREs. - -:::{typer} data_safe_haven.commands.config:config_command_group -:width: 65 -:prog: dsh config -:show-nested: -:make-sections: -::: - -## `context` commands - -`context` commands are used to manage the Data Safe Haven contexts, which are the grouping within which a single SHM and its associated SREs are organised. - -:::{typer} data_safe_haven.commands.context:context_command_group -:width: 65 -:prog: dsh context -:show-nested: -:make-sections: -::: - -## `shm` commands +The subcommands can be used to manage various aspects of a Data Safe Haven deployment. +For further detail on each subcommand, navigate to the relevant page. -`shm` commands are used to deploy or teardown DSH Safe Haven Management infrastructure +[Config](config.md) +: Management of the configuration files used to define SHMs and SREs -:::{typer} data_safe_haven.commands.shm:shm_command_group -:width: 65 -:prog: dsh shm -:show-nested: -:make-sections: -::: - -## `sre` commands - -`sre` commands are used to deploy or teardown the infrastructure for DSH Secure Research Environments - -:::{typer} data_safe_haven.commands.sre:sre_command_group -:width: 65 -:prog: dsh sre -:show-nested: -:make-sections: -::: - -## `users` commands +[Context](context.md) +: Manage DSH contexts, the groupings that encompass an SHM and its associated SREs -`users` commands are used to manage users on the Entra ID associated with a DSH deployment. +[Users](users.md) +: Management of users in Entra ID -:::{typer} data_safe_haven.commands.users:users_command_group -:width: 65 -:prog: dsh users -:show-nested: -:make-sections: -::: +[Pulumi](pulumi.md) +: An interface to the Pulumi command line interface -## `pulumi` commands +[shm](shm.md) +: Management of infrastructure for DSH Safe Haven Management environments -:::{typer} data_safe_haven.commands.pulumi:pulumi_command_group -:width: 65 -:prog: dsh pulumi -:show-nested: -:make-sections: -::: +[sre](sre.md) +: Management of infrastructure for DSH Secure Research Environments diff --git a/docs/source/reference/pulumi.md b/docs/source/reference/pulumi.md new file mode 100644 index 0000000000..1f5af340c5 --- /dev/null +++ b/docs/source/reference/pulumi.md @@ -0,0 +1,8 @@ +# `pulumi` + +:::{typer} data_safe_haven.commands.pulumi:pulumi_command_group +:width: 65 +:prog: dsh pulumi +:show-nested: +:make-sections: +::: diff --git a/docs/source/reference/shm.md b/docs/source/reference/shm.md new file mode 100644 index 0000000000..0bb9f09105 --- /dev/null +++ b/docs/source/reference/shm.md @@ -0,0 +1,10 @@ +# `shm` + +`dsh shm` commands are used to deploy or teardown DSH Safe Haven Management infrastructure + +:::{typer} data_safe_haven.commands.shm:shm_command_group +:width: 65 +:prog: dsh shm +:show-nested: +:make-sections: +::: diff --git a/docs/source/reference/sre.md b/docs/source/reference/sre.md new file mode 100644 index 0000000000..03e478efe3 --- /dev/null +++ b/docs/source/reference/sre.md @@ -0,0 +1,10 @@ +# `sre` + +`dsh sre` commands are used to deploy or teardown the infrastructure for DSH Secure Research Environments + +:::{typer} data_safe_haven.commands.sre:sre_command_group +:width: 65 +:prog: dsh sre +:show-nested: +:make-sections: +::: diff --git a/docs/source/reference/users.md b/docs/source/reference/users.md new file mode 100644 index 0000000000..9b198ce5ba --- /dev/null +++ b/docs/source/reference/users.md @@ -0,0 +1,10 @@ +# `users` + +`dsh users` commands are used to manage users on the Entra ID associated with a DSH deployment. + +:::{typer} data_safe_haven.commands.users:users_command_group +:width: 65 +:prog: dsh users +:show-nested: +:make-sections: +::: From 6c9fb1759630c300c764fbee78762154e0133481 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 09:47:20 +0000 Subject: [PATCH 44/78] Correct T2/3 PyPI/CRAN proxy information --- docs/source/overview/sensitivity_tiers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/overview/sensitivity_tiers.md b/docs/source/overview/sensitivity_tiers.md index 4aef9a32fe..995be6ab87 100644 --- a/docs/source/overview/sensitivity_tiers.md +++ b/docs/source/overview/sensitivity_tiers.md @@ -49,7 +49,7 @@ Non-technical restrictions related to information governance procedures may also - connections to the in-browser remote desktop can only be made from an agreed set of IP addresses - outbound connections to the internet from inside the environment are not possible - copy-and-paste between the environment and the user's device is not possible -- access to all packages on PyPI and CRAN is made available through a proxy or mirror server +- access to all packages on PyPI and CRAN is made available through a proxy server Non-technical restrictions related to information governance procedures may also be applied according to your organisation's needs. @@ -63,7 +63,7 @@ At the Turing connections to Tier 2 environments are only permitted from **Organ **Tier 3** environments impose the following technical controls on top of what is required at {ref}`policy_tier_2`. -- a partial replica of agreed PyPI and CRAN packages is made available through a proxy or mirror server +- an agreed subset of PyPI and CRAN packages is made available through a proxy server Non-technical restrictions related to information governance procedures may also be applied according to your organisation's needs. From 8222d646a49933dbad233cf6abe22c9a98f6a6f6 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 09:52:18 +0000 Subject: [PATCH 45/78] Add full stops --- docs/source/management/logs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 046f9c12eb..4fca60864b 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -37,10 +37,10 @@ Logs from all user workspaces are ingested into the [SRE's log analytics workspa There are three logs `Perf` -: Usage statistics for individual workspaces, such as percent memory used and percent disk space used +: Usage statistics for individual workspaces, such as percent memory used and percent disk space used. `Syslog` -: Linux system logs for individual workspaces, useful for debugging problems related to system processes +: Linux system logs for individual workspaces, useful for debugging problems related to system processes. `Heartbeat` -: Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) +: Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace). From 7088eea3f1f49848f55a71b1e79beb7c7caaac2e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 10:09:41 +0000 Subject: [PATCH 46/78] Adjust syslog description --- docs/source/management/logs.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 4fca60864b..cf55508cd3 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -40,7 +40,9 @@ There are three logs : Usage statistics for individual workspaces, such as percent memory used and percent disk space used. `Syslog` -: Linux system logs for individual workspaces, useful for debugging problems related to system processes. +: [syslog](https://www.paessler.com/it-explained/syslog) events from workspaces. +: Syslog is the _de facto_ standard protocol for logging on Linux and most applications will log to it. +: These logs will be useful for debugging problems with the workspace or workspace software. `Heartbeat` : Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace). From 168ca245117aa1ff906a1319aab4ad9c11339ccd Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 11:01:54 +0000 Subject: [PATCH 47/78] Correct blobServices URI --- data_safe_haven/infrastructure/programs/sre/data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index fcca697810..5f68a8f637 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -550,7 +550,9 @@ def __init__( } ], resource_uri=storage_account_data_private_sensitive.id.apply( - lambda resource_id: resource_id + "/blobServices" + # This is the URI of the blobServices resource which is automatically + # created. + lambda resource_id: resource_id + "/blobServices/default" ), workspace_id=props.log_analytics_workspace.id, ) From 7887e0987ea575a8c2940cdb1014a7de0bf19a68 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 11:43:12 +0000 Subject: [PATCH 48/78] Move NFSv3 accounts to a component resource Diagnostic settings are created as part of the component. --- .../infrastructure/components/__init__.py | 6 +- .../components/composite/__init__.py | 6 + .../composite/nfsv3_storage_account.py | 144 ++++++++++++++++++ .../components/wrapped/__init__.py | 2 - .../wrapped/nfsv3_storage_account.py | 79 ---------- .../programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/data.py | 70 +++------ .../programs/sre/desired_state.py | 28 ++-- 8 files changed, 191 insertions(+), 145 deletions(-) create mode 100644 data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py delete mode 100644 data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py diff --git a/data_safe_haven/infrastructure/components/__init__.py b/data_safe_haven/infrastructure/components/__init__.py index f4b93b9c3d..52043d1ad3 100644 --- a/data_safe_haven/infrastructure/components/__init__.py +++ b/data_safe_haven/infrastructure/components/__init__.py @@ -9,6 +9,8 @@ MicrosoftSQLDatabaseProps, NFSV3BlobContainerComponent, NFSV3BlobContainerProps, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, PostgresqlDatabaseComponent, PostgresqlDatabaseProps, VMComponent, @@ -23,7 +25,6 @@ ) from .wrapped import ( WrappedLogAnalyticsWorkspace, - WrappedNFSV3StorageAccount, ) __all__ = [ @@ -41,11 +42,12 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", + "NFSV3StorageAccountComponent", + "NFSV3StorageAccountProps", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "SSLCertificate", "SSLCertificateProps", "VMComponent", "WrappedLogAnalyticsWorkspace", - "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/composite/__init__.py b/data_safe_haven/infrastructure/components/composite/__init__.py index bc09bc18a8..8e561dd73a 100644 --- a/data_safe_haven/infrastructure/components/composite/__init__.py +++ b/data_safe_haven/infrastructure/components/composite/__init__.py @@ -9,6 +9,10 @@ MicrosoftSQLDatabaseProps, ) from .nfsv3_blob_container import NFSV3BlobContainerComponent, NFSV3BlobContainerProps +from .nfsv3_storage_account import ( + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, +) from .postgresql_database import PostgresqlDatabaseComponent, PostgresqlDatabaseProps from .virtual_machine import LinuxVMComponentProps, VMComponent @@ -23,6 +27,8 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", + "NFSV3StorageAccountComponent", + "NFSV3StorageAccountProps", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "VMComponent", diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py b/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py new file mode 100644 index 0000000000..ca003bbd3d --- /dev/null +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py @@ -0,0 +1,144 @@ +from collections.abc import Mapping, Sequence + +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import insights, storage + +from data_safe_haven.external import AzureIPv4Range +from data_safe_haven.infrastructure.components.wrapped import ( + WrappedLogAnalyticsWorkspace, +) +from data_safe_haven.types import AzureServiceTag + + +class NFSV3StorageAccountProps: + def __init__( + self, + account_name: Input[str], + allowed_ip_addresses: Input[Sequence[str]] | None, + allowed_service_tag: AzureServiceTag | None, + location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], + resource_group_name: Input[str], + subnet_id: Input[str], + ): + self.account_name = account_name + self.allowed_ip_addresses = allowed_ip_addresses + self.allowed_service_tag = allowed_service_tag + self.location = location + self.log_analytics_workspace = log_analytics_workspace + self.resource_group_name = resource_group_name + self.subnet_id = subnet_id + + +class NFSV3StorageAccountComponent(ComponentResource): + encryption_args = storage.EncryptionArgs( + key_source=storage.KeySource.MICROSOFT_STORAGE, + services=storage.EncryptionServicesArgs( + blob=storage.EncryptionServiceArgs( + enabled=True, key_type=storage.KeyType.ACCOUNT + ), + file=storage.EncryptionServiceArgs( + enabled=True, key_type=storage.KeyType.ACCOUNT + ), + ), + ) + + def __init__( + self, + name: str, + props: NFSV3StorageAccountProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ): + super().__init__("dsh:sre:NFSV3StorageAccountComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = {"component": "data"} | (tags if tags else {}) + + if props.allowed_service_tag == AzureServiceTag.INTERNET: + default_action = storage.DefaultAction.ALLOW + ip_rules = [] + else: + default_action = storage.DefaultAction.DENY + ip_rules = Output.from_input(props.allowed_ip_addresses).apply( + lambda ip_ranges: [ + storage.IPRuleArgs( + action=storage.Action.ALLOW, + i_p_address_or_range=str(ip_address), + ) + for ip_range in sorted(ip_ranges) + for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips() + ] + ) + + # Deploy storage account + self.storage_account = storage.StorageAccount( + f"{self._name}", + account_name=props.account_name, + allow_blob_public_access=False, + enable_https_traffic_only=True, + enable_nfs_v3=True, + encryption=self.encryption_args, + is_hns_enabled=True, + kind=storage.Kind.BLOCK_BLOB_STORAGE, + location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, + network_rule_set=storage.NetworkRuleSetArgs( + bypass=storage.Bypass.AZURE_SERVICES, + default_action=default_action, + ip_rules=ip_rules, + virtual_network_rules=[ + storage.VirtualNetworkRuleArgs( + virtual_network_resource_id=props.subnet_id, + ) + ], + ), + public_network_access=storage.PublicNetworkAccess.ENABLED, + resource_group_name=props.resource_group_name, + sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS), + opts=child_opts, + tags=child_tags, + ) + + # Add diagnostic setting for blobs + insights.DiagnosticSetting( + f"{self.storage_account._name}_diagnostic_setting", + name=f"{self.storage_account._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=self.storage_account.id.apply( + # This is the URI of the blobServices resource which is automatically + # created. + lambda resource_id: resource_id + + "/blobServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) + + self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/components/wrapped/__init__.py b/data_safe_haven/infrastructure/components/wrapped/__init__.py index b449f46859..fc5f8c8f61 100644 --- a/data_safe_haven/infrastructure/components/wrapped/__init__.py +++ b/data_safe_haven/infrastructure/components/wrapped/__init__.py @@ -1,7 +1,5 @@ from .log_analytics_workspace import WrappedLogAnalyticsWorkspace -from .nfsv3_storage_account import WrappedNFSV3StorageAccount __all__ = [ "WrappedLogAnalyticsWorkspace", - "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py b/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py deleted file mode 100644 index e259de4806..0000000000 --- a/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py +++ /dev/null @@ -1,79 +0,0 @@ -from collections.abc import Mapping, Sequence - -from pulumi import Input, Output, ResourceOptions -from pulumi_azure_native import storage - -from data_safe_haven.external import AzureIPv4Range -from data_safe_haven.types import AzureServiceTag - - -class WrappedNFSV3StorageAccount(storage.StorageAccount): - encryption_args = storage.EncryptionArgs( - key_source=storage.KeySource.MICROSOFT_STORAGE, - services=storage.EncryptionServicesArgs( - blob=storage.EncryptionServiceArgs( - enabled=True, key_type=storage.KeyType.ACCOUNT - ), - file=storage.EncryptionServiceArgs( - enabled=True, key_type=storage.KeyType.ACCOUNT - ), - ), - ) - - def __init__( - self, - resource_name: str, - *, - account_name: Input[str], - allowed_ip_addresses: Input[Sequence[str]] | None, - allowed_service_tag: AzureServiceTag | None, - location: Input[str], - resource_group_name: Input[str], - subnet_id: Input[str], - opts: ResourceOptions, - tags: Input[Mapping[str, Input[str]]], - ): - if allowed_service_tag == AzureServiceTag.INTERNET: - default_action = storage.DefaultAction.ALLOW - ip_rules = [] - else: - default_action = storage.DefaultAction.DENY - ip_rules = Output.from_input(allowed_ip_addresses).apply( - lambda ip_ranges: [ - storage.IPRuleArgs( - action=storage.Action.ALLOW, - i_p_address_or_range=str(ip_address), - ) - for ip_range in sorted(ip_ranges) - for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips() - ] - ) - - self.resource_group_name_ = Output.from_input(resource_group_name) - super().__init__( - resource_name, - account_name=account_name, - allow_blob_public_access=False, - enable_https_traffic_only=True, - enable_nfs_v3=True, - encryption=self.encryption_args, - is_hns_enabled=True, - kind=storage.Kind.BLOCK_BLOB_STORAGE, - location=location, - minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, - network_rule_set=storage.NetworkRuleSetArgs( - bypass=storage.Bypass.AZURE_SERVICES, - default_action=default_action, - ip_rules=ip_rules, - virtual_network_rules=[ - storage.VirtualNetworkRuleArgs( - virtual_network_resource_id=subnet_id, - ) - ], - ), - public_network_access=storage.PublicNetworkAccess.ENABLED, - resource_group_name=resource_group_name, - sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS), - opts=opts, - tags=tags, - ) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 96df9380b2..da796adfcc 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -377,6 +377,7 @@ def __call__(self) -> None: ldap_user_filter=ldap_user_filter, ldap_user_search_base=ldap_user_search_base, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, software_repository_hostname=user_services.software_repositories.hostname, subnet_desired_state=networking.subnet_desired_state, diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 5f68a8f637..645ea8fd87 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,7 +7,6 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, - insights, keyvault, managedidentity, network, @@ -32,10 +31,11 @@ from data_safe_haven.infrastructure.components import ( NFSV3BlobContainerComponent, NFSV3BlobContainerProps, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, SSLCertificate, SSLCertificateProps, WrappedLogAnalyticsWorkspace, - WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -471,20 +471,26 @@ def __init__( # Deploy sensitive data blob storage account # - This holds the /mnt/input and /mnt/output containers that are mounted by workspaces # - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer - storage_account_data_private_sensitive = WrappedNFSV3StorageAccount( + component_data_private_sensitive = NFSV3StorageAccountComponent( f"{self._name}_storage_account_data_private_sensitive", - # Storage account names have a maximum of 24 characters - account_name=alphanumeric( - f"{''.join(truncate_tokens(stack_name.split('-'), 11))}sensitivedata{sha256hash(self._name)}" - )[:24], - allowed_ip_addresses=data_private_sensitive_ip_addresses, - allowed_service_tag=data_private_sensitive_service_tag, - location=props.location, - subnet_id=props.subnet_data_private_id, - resource_group_name=props.resource_group_name, + NFSV3StorageAccountProps( + # Storage account names have a maximum of 24 characters + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 11))}sensitivedata{sha256hash(self._name)}" + )[:24], + allowed_ip_addresses=data_private_sensitive_ip_addresses, + allowed_service_tag=data_private_sensitive_service_tag, + location=props.location, + log_analytics_workspace=props.log_analytics_workspace, + subnet_id=props.subnet_data_private_id, + resource_group_name=props.resource_group_name, + ), opts=child_opts, tags=child_tags, ) + storage_account_data_private_sensitive = ( + component_data_private_sensitive.storage_account + ) # Deploy storage containers NFSV3BlobContainerComponent( f"{self._name}_blob_egress", @@ -516,46 +522,6 @@ def __init__( subscription_name=props.subscription_name, ), ) - # Add diagnostic setting for blobs - insights.DiagnosticSetting( - f"{storage_account_data_private_sensitive._name}_diagnostic_setting", - name=f"{storage_account_data_private_sensitive._name}_diagnostic_setting", - log_analytics_destination_type="Dedicated", - logs=[ - { - "category_group": "allLogs", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - { - "category_group": "audit", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - ], - metrics=[ - { - "category": "Transaction", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - } - ], - resource_uri=storage_account_data_private_sensitive.id.apply( - # This is the URI of the blobServices resource which is automatically - # created. - lambda resource_id: resource_id + "/blobServices/default" - ), - workspace_id=props.log_analytics_workspace.id, - ) # Set up a private endpoint for the sensitive data storage account storage_account_data_private_sensitive_endpoint = network.PrivateEndpoint( f"{storage_account_data_private_sensitive._name}_private_endpoint", diff --git a/data_safe_haven/infrastructure/programs/sre/desired_state.py b/data_safe_haven/infrastructure/programs/sre/desired_state.py index c4392f5210..20f4e357f1 100644 --- a/data_safe_haven/infrastructure/programs/sre/desired_state.py +++ b/data_safe_haven/infrastructure/programs/sre/desired_state.py @@ -31,7 +31,9 @@ from data_safe_haven.infrastructure.components import ( NFSV3BlobContainerComponent, NFSV3BlobContainerProps, - WrappedNFSV3StorageAccount, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.types import AzureDnsZoneNames @@ -55,6 +57,7 @@ def __init__( ldap_user_filter: Input[str], ldap_user_search_base: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], software_repository_hostname: Input[str], subscription_name: Input[str], @@ -73,6 +76,7 @@ def __init__( self.ldap_user_filter = ldap_user_filter self.ldap_user_search_base = ldap_user_search_base self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -102,19 +106,23 @@ def __init__( # Deploy desired state storage account # - This holds the /var/local/ansible container that is mounted by workspaces # - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer - storage_account = WrappedNFSV3StorageAccount( + storage_component = NFSV3StorageAccountComponent( f"{self._name}_storage_account", - account_name=alphanumeric( - f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}" - )[:24], - allowed_ip_addresses=props.admin_ip_addresses, - allowed_service_tag=None, - location=props.location, - resource_group_name=props.resource_group_name, - subnet_id=props.subnet_desired_state_id, + NFSV3StorageAccountProps( + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}" + )[:24], + allowed_ip_addresses=props.admin_ip_addresses, + allowed_service_tag=None, + location=props.location, + log_analytics_workspace=props.log_analytics_workspace, + resource_group_name=props.resource_group_name, + subnet_id=props.subnet_desired_state_id, + ), opts=child_opts, tags=child_tags, ) + storage_account = storage_component.storage_account # Deploy desired state share container_desired_state = NFSV3BlobContainerComponent( f"{self._name}_blob_desired_state", From a1b067fcbb7219ac1a148de0e3ffa70d3dbb8849 Mon Sep 17 00:00:00 2001 From: Matt Craddock Date: Thu, 28 Nov 2024 12:09:50 +0000 Subject: [PATCH 49/78] Update data_safe_haven/commands/users.py Co-authored-by: Jim Madge --- data_safe_haven/commands/users.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index b3a90c0664..661e3e91f0 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -153,8 +153,7 @@ def register( } usernames_to_register = [] for username in usernames: - if username in user_dict.keys(): - user_domain = user_dict[username] + if user_domain:=user_dict.get(username) if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" From 2d148498131bcb68e2226f971bebcfdf6ba6ba9e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:13:08 +0000 Subject: [PATCH 50/78] add missing colon --- data_safe_haven/commands/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 661e3e91f0..80fb1a356b 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -153,7 +153,7 @@ def register( } usernames_to_register = [] for username in usernames: - if user_domain:=user_dict.get(username) + if user_domain := user_dict.get(username): if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" From 696b192ad1c65dcb9ff4fea922d2ba5628847702 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 14:01:57 +0000 Subject: [PATCH 51/78] Add blob log documentation --- docs/source/management/logs.md | 38 +++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 80852848bd..1b6876794c 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -11,6 +11,38 @@ Select which Log Analytics Workspace you want to view by clicking on the workspa The logs can be filtered using [Kusto Query Language (KQL)](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview). +## Storage logs + +Depending on how different parts of Data Safe Haven storage are provisioned, logs may differ. + +### Sensitive data logs + +The sensitive data containers are the [ingress and egress containers](./data.md). +Logs from these containers are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageBlobLogs` +: Events occurring on the blob containers. +: For example data being uploaded, extracted or read. + +`AzureMetrics` +: Various metrics on blob container utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. + +### Desired state data logs + +The desired state container holds the data necessary to configure virtual machines in an SRE. +Logs from the desired state container are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageBlobLogs` +: Events occurring on the blob containers. +: For example data being uploaded, extracted or read. + +`AzureMetrics` +: Various metrics on blob container utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. + ## Container logs Some of the Data Safe Haven infrastructure is provisioned as containers. @@ -21,7 +53,7 @@ These include, - Gitea and Hedgedoc Logs from all containers are ingested into the [SRE's log analytics workspace](#log-workspace). -There are two logs +There are two tables, `ContainerEvents_CL` : Event logs for the container instance resources such as starting, stopping, crashes and pulling images. @@ -34,7 +66,7 @@ There are two logs Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace) using the [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/azure-monitor-agent-overview). -There are three logs +There are three tables, `Perf` : Usage statistics for individual workspaces, such as percent memory used and percent disk space used. @@ -53,7 +85,7 @@ The firewall plays a critical role in the security of a Data Safe Haven. It filters all outbound traffic through a set of FQDN rules so that each component may only reach necessary and allowed domains. Logs from the firewall are ingested into the [SREs log workspace](#log-workspace). -There are multiple tables, +There are three tables, `AZFWApplicationRule` : Logs from the firewalls FDQN filters. From 04d838faa46e78fd31cc20a4327e22b5041d1807 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 14:55:44 +0000 Subject: [PATCH 52/78] Add diagnostic setting for user data shares --- .../programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/data.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index f69fc9cd45..8cca2dff85 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -211,6 +211,7 @@ def __call__(self) -> None: dns_record=networking.shm_ns_record, dns_server_admin_password=dns.password_admin, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, storage_quota_gb_home=self.config.sre.storage_quota_gb.home, diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 711b76139f..8e1a278770 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,6 +7,7 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, + insights, keyvault, managedidentity, network, @@ -33,6 +34,7 @@ NFSV3BlobContainerProps, SSLCertificate, SSLCertificateProps, + WrappedLogAnalyticsWorkspace, WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -51,6 +53,7 @@ def __init__( dns_record: Input[network.RecordSet], dns_server_admin_password: Input[pulumi_random.RandomPassword], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], storage_quota_gb_home: Input[int], @@ -69,6 +72,7 @@ def __init__( self.dns_record = dns_record self.password_dns_server_admin = dns_server_admin_password self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -615,6 +619,47 @@ def __init__( opts=child_opts, tags=child_tags, ) + # Add diagnostic setting for files + insights.DiagnosticSetting( + f"{storage_account_data_private_user._name}_diagnostic_setting", + name=f"{storage_account_data_private_user._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_private_user.id.apply( + # This is the URI of the fileServices resource which is automatically + # created. + lambda resource_id: resource_id + + "/fileServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) storage.FileShare( f"{storage_account_data_private_user._name}_files_home", access_tier=storage.ShareAccessTier.PREMIUM, From b90a1ff9342f12f665a071fe0b2e04c792b9978d Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 15:03:12 +0000 Subject: [PATCH 53/78] Add diagnostic setting for config file shares --- .../infrastructure/programs/sre/data.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 8e1a278770..0dabd47f71 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -425,6 +425,47 @@ def __init__( resource_group_name=kwargs["resource_group_name"], ) ) + # Add diagnostic setting for files + insights.DiagnosticSetting( + f"{storage_account_data_configuration._name}_diagnostic_setting", + name=f"{storage_account_data_configuration._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_configuration.id.apply( + # This is the URI of the fileService resource which is automatically + # created. + lambda resource_id: resource_id + + "/fileServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) # Set up a private endpoint for the configuration data storage account storage_account_data_configuration_private_endpoint = network.PrivateEndpoint( f"{storage_account_data_configuration._name}_private_endpoint", From 2e56565d77c243b8113a397f0d06e9a5463ebdcc Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:55:55 +0000 Subject: [PATCH 54/78] Add documentation for file share logs --- docs/source/management/logs.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 1b6876794c..408ec9b631 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -43,6 +43,35 @@ There are two tables, : Various metrics on blob container utilisation and performance. : This table is not reserved for the firewall and other resources may log to it. +### User data logs + +The user data file share holds the {ref}`role_researcher`s' [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. +Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageFileLogs` +: NFS events occurring on the file share. +: For example data being written or directories being accessed + +`AzureMetrics` +: Various metrics on file share utilisation and performance. +: This table is not reserved for the user data share and other resources may log to it. + +### Configuration data logs + +There are multiple configuration data file shares. +Each contains the configuration and state data for the Data Safe Haven [services deployed as containers](#container-logs). +Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageFileLogs` +: SMB events occurring on the file share. +: For example data being written or directories being accessed + +`AzureMetrics` +: Various metrics on file share utilisation and performance. +: This table is not reserved for the configuration data shares and other resources may log to it. + ## Container logs Some of the Data Safe Haven infrastructure is provisioned as containers. From 799f96b7944145e40d4c7889bd30f5374c6cf4a9 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:56:55 +0000 Subject: [PATCH 55/78] Correct descriptions --- docs/source/management/logs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 408ec9b631..63314654fc 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -27,7 +27,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the sensitive data containers and other resources may log to it. ### Desired state data logs @@ -41,7 +41,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the desired state data container and other resources may log to it. ### User data logs From 37fdeb7e04d82eb42c4fd08e6b5a83f522692e4e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:56:55 +0000 Subject: [PATCH 56/78] Correct descriptions --- docs/source/management/logs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 1b6876794c..d7607cf98a 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -27,7 +27,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the sensitive data containers and other resources may log to it. ### Desired state data logs @@ -41,7 +41,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the desired state data container and other resources may log to it. ## Container logs From d5802fb893a0d56ab2256229704ab5ce316c1917 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:03:51 +0000 Subject: [PATCH 57/78] Improve reference rendering --- docs/source/management/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 63314654fc..10b9bfb0e5 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -45,7 +45,7 @@ There are two tables, ### User data logs -The user data file share holds the {ref}`role_researcher`s' [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. +The user data file share holds the {ref}`researchers'` [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). There are two tables, From da382e0e600991e4e19728760d5b4e048699a3d1 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:15:59 +0000 Subject: [PATCH 58/78] Use Output concat method Co-authored-by: James Robinson --- .../infrastructure/programs/sre/data.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index e348f25976..a789d61ad0 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -459,12 +459,8 @@ def __init__( }, } ], - resource_uri=storage_account_data_configuration.id.apply( - # This is the URI of the fileService resource which is automatically - # created. - lambda resource_id: resource_id - + "/fileServices/default" - ), + # This is the URI of the automatically created fileService resource + resource_uri=Output.concat(storage_account_data_configuration.id, "/fileServices/default"), workspace_id=props.log_analytics_workspace.id, ) # Set up a private endpoint for the configuration data storage account @@ -700,12 +696,8 @@ def __init__( }, } ], - resource_uri=storage_account_data_private_user.id.apply( - # This is the URI of the fileServices resource which is automatically - # created. - lambda resource_id: resource_id - + "/fileServices/default" - ), + # This is the URI of the automatically created fileService resource + resource_uri=Output.concat(storage_account_data_private_user.id, "/fileServices/default"), workspace_id=props.log_analytics_workspace.id, ) storage.FileShare( From d73704a0742402f638430e6c321e2872f1ff7bb4 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:18:14 +0000 Subject: [PATCH 59/78] Fix linting --- data_safe_haven/infrastructure/programs/sre/data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index a789d61ad0..825861c122 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -460,7 +460,9 @@ def __init__( } ], # This is the URI of the automatically created fileService resource - resource_uri=Output.concat(storage_account_data_configuration.id, "/fileServices/default"), + resource_uri=Output.concat( + storage_account_data_configuration.id, "/fileServices/default" + ), workspace_id=props.log_analytics_workspace.id, ) # Set up a private endpoint for the configuration data storage account @@ -697,7 +699,9 @@ def __init__( } ], # This is the URI of the automatically created fileService resource - resource_uri=Output.concat(storage_account_data_private_user.id, "/fileServices/default"), + resource_uri=Output.concat( + storage_account_data_private_user.id, "/fileServices/default" + ), workspace_id=props.log_analytics_workspace.id, ) storage.FileShare( From 668008124f1e1583b9d8184c9af08fa93f0dad45 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:05:52 +0000 Subject: [PATCH 60/78] remove path insertion from sphinx config --- docs/source/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index f2d17d4d84..a1e2c34385 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -5,9 +5,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html import datetime import emoji -import sys -import os -sys.path.insert(0, os.path.abspath(os.path.join("..", ".."))) + # -- Project information ----------------------------------------------------- From 90d8f01c58467644aba147477d5c37b57cf9a0ce Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:06:23 +0000 Subject: [PATCH 61/78] Add description of pulumi function --- docs/source/reference/pulumi.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/reference/pulumi.md b/docs/source/reference/pulumi.md index 1f5af340c5..e6cb051860 100644 --- a/docs/source/reference/pulumi.md +++ b/docs/source/reference/pulumi.md @@ -1,5 +1,7 @@ # `pulumi` +The `pulumi` function allows you to run native Pulumi command line functions to interact with an SRE stack + :::{typer} data_safe_haven.commands.pulumi:pulumi_command_group :width: 65 :prog: dsh pulumi From 1b89bf6d0fb397436c4a8c5e51a7a7860c87ea30 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:39:14 +0000 Subject: [PATCH 62/78] Change section name to CLI Reference --- docs/source/index.md | 4 ++-- docs/source/reference/index.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/index.md b/docs/source/index.md index 64588350d7..2c3be84b84 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -43,8 +43,8 @@ You can read them through in order or simply jump to the section that you are mo - If you want to deploy your own Data Safe Haven. - [**Management**](management/index.md) - If you want to manage a Data Safe Haven that you (or someone else) has deployed. -- [**Reference**](reference/index.md) - - If you want details of all the Command Line Interface +- [**CLI Reference**](reference/index.md) + - If you want details of all the Command Line Interface commands. - [**Roles**](roles/index.md) - Information about the different user roles in the Data Safe Haven. - Instructions and advice for the actions of different user roles. diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index d790c7d448..6bcb99fd2e 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -1,4 +1,4 @@ -# Reference +# CLI Reference :::{toctree} :hidden: From 12b599ed0c4bf301e499cfbb36987882b3dc784a Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:39:33 +0000 Subject: [PATCH 63/78] Cross reference user commands --- docs/source/management/user.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/management/user.md b/docs/source/management/user.md index d996321162..337af2302f 100644 --- a/docs/source/management/user.md +++ b/docs/source/management/user.md @@ -7,7 +7,7 @@ You will need a full name, phone number, email address and country for each user ::: 1. You can add users directly in your Entra tenant, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-create-delete-users). -1. Alternatively, you can add multiple users from a CSV file with columns named (`GivenName`, `Surname`, `Phone`, `Email`, `CountryCode`). +1. Alternatively, you can use {typer}`dsh users add` to add multiple users from a CSV file with columns named (`GivenName`, `Surname`, `Phone`, `Email`, `CountryCode`). - (Optional) you can provide a `Domain` column if you like but this will otherwise default to the domain of your SHM - {{warning}} **Phone** must be in [E.123 international format](https://en.wikipedia.org/wiki/E.123) - {{warning}} **CountryCode** is the two letter [ISO 3166-1 Alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements) code for the country where the user is based @@ -32,7 +32,7 @@ $ dsh users add PATH_TO_MY_CSV_FILE 1. Browse to **{menuselection}`Groups --> All Groups`** 1. Click on the group named **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** 1. Browse to **{menuselection}`Manage --> Members`** from the secondary menu on the left side -- You can do this at the command line by running the following command: +- You can do this at the command line by running `dsh users list`: ```{code} shell $ dsh users list YOUR_SRE_NAME @@ -54,7 +54,7 @@ $ dsh users add PATH_TO_MY_CSV_FILE ## Assign existing users to an SRE 1. You can do this directly in your Entra tenant by adding them to the **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** group, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/groups-view-azure-portal#add-a-group-member). -1. Alternatively, you can add multiple users from the command line: +1. Alternatively, you can add multiple users from the command line using {typer}`dsh users register`: ```{code} shell $ dsh users register YOUR_SRE_NAME -u USERNAME_1 -u USERNAME_2 @@ -69,7 +69,7 @@ $ dsh users add PATH_TO_MY_CSV_FILE ## Manually register users for self-service password reset :::{tip} -Users created via the `dsh users` command line tool will be automatically registered for SSPR. +Users created via the {typer}`dsh users` command line tool will be automatically registered for SSPR. ::: If you have manually created a user and want to enable SSPR, do the following From 5be2d7111c3ea2f5cdc80c02e0f2fcf6d84ea98c Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:39:47 +0000 Subject: [PATCH 64/78] cross reference commands in sre management --- docs/source/management/sre.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/management/sre.md b/docs/source/management/sre.md index aab94c31a1..fddaa5e135 100644 --- a/docs/source/management/sre.md +++ b/docs/source/management/sre.md @@ -2,13 +2,13 @@ ## List available SRE configurations and deployment status -- Run the following if you want to check what SRE configurations are available in the current context, and whether those SREs are deployed +- Use {typer}`dsh config available` to check what SRE configurations are available in the current context, and whether those SREs are deployed. ```{code} shell $ dsh config available ``` -which will give output like the following +will give output like the following ```{code} shell Available SRE configurations for context 'green': @@ -23,7 +23,7 @@ Available SRE configurations for context 'green': ## Remove a deployed Data Safe Haven -- Run the following if you want to teardown a deployed SRE: +- Use {typer}`dsh sre teardown` to teardown a deployed SRE: ```{code} shell $ dsh sre teardown YOUR_SRE_NAME @@ -37,7 +37,7 @@ Ensure that any desired outputs have been extracted before deleting the SRE. The user groups for the SRE on Microsoft Entra ID will also be deleted. :::: -- Run the following if you want to teardown the deployed SHM: +- Use {typer}`dsh shm teardown` if you want to teardown the deployed SHM: ```{code} shell $ dsh shm teardown @@ -54,7 +54,7 @@ All SREs associated with the SHM should be torn down before the SHM is torn down SREs are modified by updating the configuration then running the deploy command. -- The existing configuration for the SRE can be shown using the following: +- The existing configuration for the SRE can be shown using {typer}`dsh config show`: ```{code} shell $ dsh config show YOUR_SRE_NAME @@ -66,14 +66,14 @@ $ dsh config show YOUR_SRE_NAME $ dsh config show YOUR_SRE_NAME --file YOUR_SRE_NAME.yaml ``` -- Edit the configuration file locally, and upload the new version: +- Edit the configuration file locally, and upload the new version using {typer}`dsh config upload`: ```{code} shell $ dsh config upload YOUR_SRE_NAME.yaml ``` - You will be shown the differences between the existing configuration and the new configuration and asked to confirm that they are correct. -- Finally, deploy your SRE to apply any changes: +- Finally, deploy your SRE using {typer}`dsh sre deploy` to apply any changes: ```{code} shell $ dsh sre deploy YOUR_SRE_NAME From 1f27626365357de5cebb349f8c731438fcdf27d2 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:39:57 +0000 Subject: [PATCH 65/78] cross reference commands for SRE deployment --- docs/source/deployment/deploy_sre.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/deployment/deploy_sre.md b/docs/source/deployment/deploy_sre.md index 5a5a5b4166..ebf1aa425a 100644 --- a/docs/source/deployment/deploy_sre.md +++ b/docs/source/deployment/deploy_sre.md @@ -21,6 +21,7 @@ $ az provider register --name Microsoft.Network Each project will have its own dedicated SRE. - Create a configuration file (optionally starting from one of our standard {ref}`policy_classification_sensitivity_tiers`) +- The {typer}`dsh-config-template` command provides template configuration files ::::{admonition} EITHER start from a blank template :class: dropdown note @@ -254,7 +255,7 @@ If you want to make changes to the config, edit this file and then run `dsh conf ## Deployment -- Deploy each SRE individually [approx 30 minutes]: +- Deploy each SRE individually using {typer}`dsh sre deploy` [approx 30 minutes]: :::{code} shell $ dsh sre deploy YOUR_SRE_NAME From b9510518b9624c84bb074ac768cec347f070d09f Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 29 Nov 2024 16:40:09 +0000 Subject: [PATCH 66/78] cross reference commands for shm deployment --- docs/source/deployment/deploy_shm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment/deploy_shm.md b/docs/source/deployment/deploy_shm.md index b26d451bfb..ef6b7090f5 100644 --- a/docs/source/deployment/deploy_shm.md +++ b/docs/source/deployment/deploy_shm.md @@ -54,7 +54,7 @@ Before deploying the Safe Haven Management (SHM) infrastructure you need to deci Use the short name without spaces, _e.g._ **uksouth** not **UK South** ::: -Once you've decided on these, run the following command: [approx 5 minutes]: +Once you've decided on these, run the {typer}`dsh-shm-deploy` command: [approx 5 minutes]: :::{code} shell $ dsh shm deploy --entra-tenant-id YOUR_ENTRA_TENANT_ID \ From 4165cab3824a8c3d8a83f09b631dc86cc27baa67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 03:23:50 +0000 Subject: [PATCH 67/78] Bump the production-dependencies group with 9 updates Bumps the production-dependencies group with 9 updates: | Package | From | To | | --- | --- | --- | | [cryptography](https://github.com/pyca/cryptography) | `43.0.3` | `44.0.0` | | [pulumi-azure-native](https://github.com/pulumi/pulumi-azure-native) | `2.73.1` | `2.74.0` | | [pulumi](https://github.com/pulumi/pulumi) | `3.141.0` | `3.142.0` | | [pydantic](https://github.com/pydantic/pydantic) | `2.10.1` | `2.10.2` | | [pyjwt[crypto]](https://github.com/jpadilla/pyjwt) | `2.10.0` | `2.10.1` | | [typer](https://github.com/fastapi/typer) | `0.13.1` | `0.14.0` | | [pandas-stubs](https://github.com/pandas-dev/pandas-stubs) | `2.2.3.241009` | `2.2.3.241126` | | [ruff](https://github.com/astral-sh/ruff) | `0.8.0` | `0.8.1` | | [pytest](https://github.com/pytest-dev/pytest) | `8.3.3` | `8.3.4` | Updates `cryptography` from 43.0.3 to 44.0.0 - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/43.0.3...44.0.0) Updates `pulumi-azure-native` from 2.73.1 to 2.74.0 - [Release notes](https://github.com/pulumi/pulumi-azure-native/releases) - [Changelog](https://github.com/pulumi/pulumi-azure-native/blob/master/CHANGELOG_OLD.md) - [Commits](https://github.com/pulumi/pulumi-azure-native/compare/v2.73.1...v2.74.0) Updates `pulumi` from 3.141.0 to 3.142.0 - [Release notes](https://github.com/pulumi/pulumi/releases) - [Changelog](https://github.com/pulumi/pulumi/blob/master/CHANGELOG.md) - [Commits](https://github.com/pulumi/pulumi/compare/v3.141.0...v3.142.0) Updates `pydantic` from 2.10.1 to 2.10.2 - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.10.1...v2.10.2) Updates `pyjwt[crypto]` from 2.10.0 to 2.10.1 - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.10.0...2.10.1) Updates `typer` from 0.13.1 to 0.14.0 - [Release notes](https://github.com/fastapi/typer/releases) - [Changelog](https://github.com/fastapi/typer/blob/master/docs/release-notes.md) - [Commits](https://github.com/fastapi/typer/compare/0.13.1...0.14.0) Updates `pandas-stubs` from 2.2.3.241009 to 2.2.3.241126 - [Changelog](https://github.com/pandas-dev/pandas-stubs/blob/main/docs/release_procedure.md) - [Commits](https://github.com/pandas-dev/pandas-stubs/compare/v2.2.3.241009...v2.2.3.241126) Updates `ruff` from 0.8.0 to 0.8.1 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.8.0...0.8.1) Updates `pytest` from 8.3.3 to 8.3.4 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...8.3.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major dependency-group: production-dependencies - dependency-name: pulumi-azure-native dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pulumi dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: pyjwt[crypto] dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: typer dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pandas-stubs dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies ... Signed-off-by: dependabot[bot] --- pyproject.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 49099b230d..dc9f8d41dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,20 +42,20 @@ dependencies = [ "azure-storage-file-datalake==12.18.0", "azure-storage-file-share==12.20.0", "chevron==0.14.0", - "cryptography==43.0.3", + "cryptography==44.0.0", "fqdn==1.5.1", "psycopg[binary]==3.1.19", # needed for installation on older MacOS versions - "pulumi-azure-native==2.73.1", + "pulumi-azure-native==2.74.0", "pulumi-azuread==6.0.1", "pulumi-random==4.16.7", - "pulumi==3.141.0", - "pydantic==2.10.1", - "pyjwt[crypto]==2.10.0", + "pulumi==3.142.0", + "pydantic==2.10.2", + "pyjwt[crypto]==2.10.1", "pytz==2024.2", "pyyaml==6.0.2", "rich==13.9.4", "simple-acme-dns==3.2.0", - "typer==0.13.1", + "typer==0.14.0", "websocket-client==1.8.0", ] @@ -77,9 +77,9 @@ lint = [ "ansible==11.0.0", "black==24.10.0", "mypy==1.13.0", - "pandas-stubs==2.2.3.241009", - "pydantic==2.10.1", - "ruff==0.8.0", + "pandas-stubs==2.2.3.241126", + "pydantic==2.10.2", + "ruff==0.8.1", "types-appdirs==1.4.3.5", "types-chevron==0.14.2.20240310", "types-pytz==2024.2.0.20241003", @@ -90,7 +90,7 @@ test = [ "coverage==7.6.8", "freezegun==1.5.1", "pytest-mock==3.14.0", - "pytest==8.3.3", + "pytest==8.3.4", "requests-mock==1.12.1", ] From 04d5802730ca46c9474b87d4d1cb690290e022ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 2 Dec 2024 03:30:33 +0000 Subject: [PATCH 68/78] [dependabot skip] :wrench: Update Python requirements files --- .hatch/requirements-lint.txt | 20 ++++++++++---------- .hatch/requirements-test.txt | 32 ++++++++++++++++---------------- .hatch/requirements.txt | 26 +++++++++++++------------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.hatch/requirements-lint.txt b/.hatch/requirements-lint.txt index e1ab89f54e..c09dc3ebb9 100644 --- a/.hatch/requirements-lint.txt +++ b/.hatch/requirements-lint.txt @@ -5,9 +5,9 @@ # - ansible==11.0.0 # - black==24.10.0 # - mypy==1.13.0 -# - pandas-stubs==2.2.3.241009 -# - pydantic==2.10.1 -# - ruff==0.8.0 +# - pandas-stubs==2.2.3.241126 +# - pydantic==2.10.2 +# - ruff==0.8.1 # - types-appdirs==1.4.3.5 # - types-chevron==0.14.2.20240310 # - types-pytz==2024.2.0.20241003 @@ -81,7 +81,7 @@ click-help-colors==0.9.4 # via molecule colorama==0.4.6 # via tox -cryptography==43.0.3 +cryptography==44.0.0 # via ansible-core distlib==0.3.9 # via @@ -151,7 +151,7 @@ packaging==24.2 # pytest # pytest-ansible # tox -pandas-stubs==2.2.3.241009 +pandas-stubs==2.2.3.241126 # via hatch.envs.lint parsley==1.3 # via bindep @@ -178,7 +178,7 @@ ptyprocess==0.7.0 # via pexpect pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via hatch.envs.lint pydantic-core==2.27.1 # via pydantic @@ -186,7 +186,7 @@ pygments==2.18.0 # via rich pyproject-api==1.8.0 # via tox -pytest==8.3.3 +pytest==8.3.4 # via # pytest-ansible # pytest-xdist @@ -197,7 +197,7 @@ pytest-ansible==24.9.0 # tox-ansible pytest-xdist==3.6.1 # via tox-ansible -python-daemon==3.1.0 +python-daemon==3.1.1 # via ansible-runner python-gnupg==0.5.3 # via ansible-sign @@ -233,7 +233,7 @@ ruamel-yaml==0.18.6 # via ansible-lint ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.1 # via hatch.envs.lint subprocess-tee==0.4.2 # via @@ -265,7 +265,7 @@ tzdata==2024.2 # via ansible-navigator urllib3==2.2.3 # via types-requests -virtualenv==20.27.1 +virtualenv==20.28.0 # via tox wcmatch==10.0 # via diff --git a/.hatch/requirements-test.txt b/.hatch/requirements-test.txt index 8c95d7dce0..ce24e1caee 100644 --- a/.hatch/requirements-test.txt +++ b/.hatch/requirements-test.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.12 # -# [constraints] .hatch/requirements.txt (SHA256: 3586aa93da255077aac182009c06aa28b96ec15387beec4148e3bebd2b9f8852) +# [constraints] .hatch/requirements.txt (SHA256: 9b78097f41c11566a80e32726aefa74a983ac227fce27db9adba04ae7594da1c) # # - appdirs==1.4.4 # - azure-core==1.32.0 @@ -21,25 +21,25 @@ # - azure-storage-file-datalake==12.18.0 # - azure-storage-file-share==12.20.0 # - chevron==0.14.0 -# - cryptography==43.0.3 +# - cryptography==44.0.0 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.73.1 +# - pulumi-azure-native==2.74.0 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.141.0 -# - pydantic==2.10.1 -# - pyjwt[crypto]==2.10.0 +# - pulumi==3.142.0 +# - pydantic==2.10.2 +# - pyjwt[crypto]==2.10.1 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.1 +# - typer==0.14.0 # - websocket-client==1.8.0 # - coverage==7.6.8 # - freezegun==1.5.1 # - pytest-mock==3.14.0 -# - pytest==8.3.3 +# - pytest==8.3.4 # - requests-mock==1.12.1 # @@ -182,7 +182,7 @@ click==8.1.7 # typer coverage==7.6.8 # via hatch.envs.test -cryptography==43.0.3 +cryptography==44.0.0 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -295,14 +295,14 @@ psycopg-binary==3.1.19 # via # -c .hatch/requirements.txt # psycopg -pulumi==3.141.0 +pulumi==3.142.0 # via # -c .hatch/requirements.txt # hatch.envs.test # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.73.1 +pulumi-azure-native==2.74.0 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -318,7 +318,7 @@ pycparser==2.22 # via # -c .hatch/requirements.txt # cffi -pydantic==2.10.1 +pydantic==2.10.2 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -330,12 +330,12 @@ pygments==2.18.0 # via # -c .hatch/requirements.txt # rich -pyjwt==2.10.0 +pyjwt==2.10.1 # via # -c .hatch/requirements.txt # hatch.envs.test # msal -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # -c .hatch/requirements.txt # acme @@ -344,7 +344,7 @@ pyrfc3339==2.0.1 # via # -c .hatch/requirements.txt # acme -pytest==8.3.3 +pytest==8.3.4 # via # hatch.envs.test # pytest-mock @@ -403,7 +403,7 @@ six==1.16.0 # azure-core # pulumi # python-dateutil -typer==0.13.1 +typer==0.14.0 # via # -c .hatch/requirements.txt # hatch.envs.test diff --git a/.hatch/requirements.txt b/.hatch/requirements.txt index 82ad061fc0..f2589f1f68 100644 --- a/.hatch/requirements.txt +++ b/.hatch/requirements.txt @@ -19,20 +19,20 @@ # - azure-storage-file-datalake==12.18.0 # - azure-storage-file-share==12.20.0 # - chevron==0.14.0 -# - cryptography==43.0.3 +# - cryptography==44.0.0 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.73.1 +# - pulumi-azure-native==2.74.0 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.141.0 -# - pydantic==2.10.1 -# - pyjwt[crypto]==2.10.0 +# - pulumi==3.142.0 +# - pydantic==2.10.2 +# - pyjwt[crypto]==2.10.1 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.1 +# - typer==0.14.0 # - websocket-client==1.8.0 # @@ -122,7 +122,7 @@ chevron==0.14.0 # via hatch.envs.default click==8.1.7 # via typer -cryptography==43.0.3 +cryptography==44.0.0 # via # hatch.envs.default # acme @@ -192,13 +192,13 @@ psycopg==3.1.19 # via hatch.envs.default psycopg-binary==3.1.19 # via psycopg -pulumi==3.141.0 +pulumi==3.142.0 # via # hatch.envs.default # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.73.1 +pulumi-azure-native==2.74.0 # via hatch.envs.default pulumi-azuread==6.0.1 # via hatch.envs.default @@ -206,17 +206,17 @@ pulumi-random==4.16.7 # via hatch.envs.default pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via hatch.envs.default pydantic-core==2.27.1 # via pydantic pygments==2.18.0 # via rich -pyjwt==2.10.0 +pyjwt==2.10.1 # via # hatch.envs.default # msal -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # acme # josepy @@ -257,7 +257,7 @@ six==1.16.0 # via # azure-core # pulumi -typer==0.13.1 +typer==0.14.0 # via hatch.envs.default typing-extensions==4.12.2 # via From 24213888ae05a085980e77e9bcc1f242024a7115 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 03:34:22 +0000 Subject: [PATCH 69/78] Bump karancode/yamllint-github-action from 2.1.1 to 3.0.0 Bumps [karancode/yamllint-github-action](https://github.com/karancode/yamllint-github-action) from 2.1.1 to 3.0.0. - [Release notes](https://github.com/karancode/yamllint-github-action/releases) - [Commits](https://github.com/karancode/yamllint-github-action/compare/v2.1.1...v3.0.0) --- updated-dependencies: - dependency-name: karancode/yamllint-github-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint_code.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_code.yaml b/.github/workflows/lint_code.yaml index 7786fc4b62..4d0caed16c 100644 --- a/.github/workflows/lint_code.yaml +++ b/.github/workflows/lint_code.yaml @@ -108,7 +108,7 @@ jobs: done rm expanded.tmp - name: Lint YAML - uses: karancode/yamllint-github-action@v2.1.1 + uses: karancode/yamllint-github-action@v3.0.0 with: yamllint_strict: true yamllint_comment: false From c64c7d2bc6828fdc2c338f98940c97488d8d7f93 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:30:40 +0000 Subject: [PATCH 70/78] Split multi-line error message between stdout and log --- data_safe_haven/commands/users.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 80fb1a356b..8c8b232ceb 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -5,6 +5,7 @@ import typer +from data_safe_haven import console from data_safe_haven.administration.users import UserHandler from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError @@ -155,18 +156,20 @@ def register( for username in usernames: if user_domain := user_dict.get(username): if shm_config.shm.fqdn not in user_domain: + console.print( + f"User [green]'{username}[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" + f"SRE [yellow]'{sre}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue]." + ) logger.error( - f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" - f"SRE [yellow]'{sre}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" "The user's principal domain name must match the domain of the SRE to be registered." ) else: usernames_to_register.append(username) else: logger.error( - f"Username '{username}' does not belong to this Data Safe Haven deployment.\n" - "Please use 'dsh users add' to create this user." + f"Username '{username}' does not belong to this Data Safe Haven deployment." ) + console.print("Please use 'dsh users add' to create this user.") users.register(sre_config.name, usernames_to_register) except DataSafeHavenError as exc: logger.critical(f"Could not register Data Safe Haven users with SRE '{sre}'.") @@ -270,8 +273,8 @@ def unregister( else: logger.error( f"Username '{username}' does not belong to this Data Safe Haven deployment." - " Please use 'dsh users add' to create it." ) + console.print("Please use 'dsh users add' to create it.") for group_name in ( f"{sre_config.name} Users", f"{sre_config.name} Privileged Users", From bd8024270f247ca2ab03551df270aee9220b78bb Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:42:20 +0000 Subject: [PATCH 71/78] update requirements-docs.txt --- .hatch/requirements-docs.txt | 37 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/.hatch/requirements-docs.txt b/.hatch/requirements-docs.txt index 2a11901466..d8350ffbe4 100644 --- a/.hatch/requirements-docs.txt +++ b/.hatch/requirements-docs.txt @@ -19,20 +19,20 @@ # - azure-storage-file-datalake==12.18.0 # - azure-storage-file-share==12.20.0 # - chevron==0.14.0 -# - cryptography==43.0.3 +# - cryptography==44.0.0 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.73.1 +# - pulumi-azure-native==2.74.0 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.141.0 -# - pydantic==2.10.1 -# - pyjwt[crypto]==2.10.0 +# - pulumi==3.142.0 +# - pydantic==2.10.2 +# - pyjwt[crypto]==2.10.1 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.1 +# - typer==0.14.0 # - websocket-client==1.8.0 # - emoji==2.14.0 # - myst-parser==4.0.0 @@ -90,7 +90,7 @@ azure-mgmt-compute==33.0.0 # via hatch.envs.docs azure-mgmt-containerinstance==10.1.0 # via hatch.envs.docs -azure-mgmt-core==1.4.0 +azure-mgmt-core==1.5.0 # via # azure-mgmt-compute # azure-mgmt-containerinstance @@ -140,7 +140,7 @@ click==8.1.7 # via # typer # typer-slim -cryptography==43.0.3 +cryptography==44.0.0 # via # hatch.envs.docs # acme @@ -152,7 +152,7 @@ cryptography==43.0.3 # msal # pyjwt # pyopenssl -debugpy==1.8.7 +debugpy==1.8.9 # via pulumi dill==0.3.9 # via pulumi @@ -206,7 +206,7 @@ mdit-py-plugins==0.4.2 # via myst-parser mdurl==0.1.2 # via markdown-it-py -msal==1.31.0 +msal==1.31.1 # via # azure-identity # msal-extensions @@ -235,13 +235,13 @@ psycopg==3.1.19 # via hatch.envs.docs psycopg-binary==3.1.19 # via psycopg -pulumi==3.141.0 +pulumi==3.142.0 # via # hatch.envs.docs # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.73.1 +pulumi-azure-native==2.74.0 # via hatch.envs.docs pulumi-azuread==6.0.1 # via hatch.envs.docs @@ -249,7 +249,7 @@ pulumi-random==4.16.7 # via hatch.envs.docs pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via hatch.envs.docs pydantic-core==2.27.1 # via pydantic @@ -261,21 +261,20 @@ pygments==2.18.0 # pydata-sphinx-theme # rich # sphinx -pyjwt==2.10.0 +pyjwt==2.10.1 # via # hatch.envs.docs # msal -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # acme # josepy -pyrfc3339==1.1 +pyrfc3339==2.0.1 # via acme pytz==2024.2 # via # hatch.envs.docs # acme - # pyrfc3339 pyyaml==6.0.2 # via # hatch.envs.docs @@ -339,9 +338,9 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sphinxcontrib-typer==0.5.1 # via hatch.envs.docs -typer==0.13.1 +typer==0.14.0 # via hatch.envs.docs -typer-slim==0.12.5 +typer-slim==0.14.0 # via sphinxcontrib-typer typing-extensions==4.12.2 # via From 8ef1d8aedc403b6ec1f4c6370e92e6a5300ad7a6 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:51:49 +0000 Subject: [PATCH 72/78] Add path back to sphinx configuration --- docs/source/conf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a1e2c34385..82a1e4bb0b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -5,7 +5,9 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html import datetime import emoji - +from pathlib import Path +import sys +sys.path.insert(0, Path(__file__).parent.parent.absolute()) # -- Project information ----------------------------------------------------- From 2438b6e9e70e46c0de749b2ccc7b1160824363ff Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:13:37 +0000 Subject: [PATCH 73/78] modify path in conf.py --- docs/source/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 82a1e4bb0b..a1fc04ee68 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -7,7 +7,8 @@ import emoji from pathlib import Path import sys -sys.path.insert(0, Path(__file__).parent.parent.absolute()) +sys.path.insert(0, Path(__file__).parents[2].absolute()) +print(sys.path) # -- Project information ----------------------------------------------------- From beab3c765ce113c6f884d33f0de9d5b6afefbc77 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:29:18 +0000 Subject: [PATCH 74/78] modify path --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a1fc04ee68..cd20fee83a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -7,7 +7,7 @@ import emoji from pathlib import Path import sys -sys.path.insert(0, Path(__file__).parents[2].absolute()) +sys.path.insert(0, Path(__file__).parents[1].absolute()) print(sys.path) # -- Project information ----------------------------------------------------- From b80e78ee84dc63eae37716ae5e22bbb0a5d78b6c Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:43:50 +0000 Subject: [PATCH 75/78] remove path --- docs/source/conf.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index cd20fee83a..a1e2c34385 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -5,10 +5,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html import datetime import emoji -from pathlib import Path -import sys -sys.path.insert(0, Path(__file__).parents[1].absolute()) -print(sys.path) + # -- Project information ----------------------------------------------------- From 8a3f1c933843cfcd433b10b4948a52f35db76449 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 15:44:06 +0000 Subject: [PATCH 76/78] install package on readthedocs --- .readthedocs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 145d005f83..56f8f2b24f 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -16,6 +16,8 @@ formats: python: install: - requirements: .hatch/requirements-docs.txt + - method: pip + path: . sphinx: configuration: docs/source/conf.py From d1fd417879b51b6ab04f9b439a43293df77aac15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 22:00:51 +0000 Subject: [PATCH 77/78] Bump ansible-core from 2.18.0 to 2.18.1 in /.hatch Bumps [ansible-core](https://github.com/ansible/ansible) from 2.18.0 to 2.18.1. - [Release notes](https://github.com/ansible/ansible/releases) - [Commits](https://github.com/ansible/ansible/compare/v2.18.0...v2.18.1) --- updated-dependencies: - dependency-name: ansible-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .hatch/requirements-lint.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.hatch/requirements-lint.txt b/.hatch/requirements-lint.txt index c09dc3ebb9..9fc366c627 100644 --- a/.hatch/requirements-lint.txt +++ b/.hatch/requirements-lint.txt @@ -29,7 +29,7 @@ ansible-compat==24.10.0 # ansible-lint # molecule # pytest-ansible -ansible-core==2.18.0 +ansible-core==2.18.1 # via # ansible # ansible-compat From c744946c6aa3755af1c57ded216ee67b992de55c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Tue, 3 Dec 2024 22:08:20 +0000 Subject: [PATCH 78/78] [dependabot skip] :wrench: Update Python requirements files --- .hatch/requirements-lint.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.hatch/requirements-lint.txt b/.hatch/requirements-lint.txt index 9fc366c627..0c10259677 100644 --- a/.hatch/requirements-lint.txt +++ b/.hatch/requirements-lint.txt @@ -197,7 +197,7 @@ pytest-ansible==24.9.0 # tox-ansible pytest-xdist==3.6.1 # via tox-ansible -python-daemon==3.1.1 +python-daemon==3.1.2 # via ansible-runner python-gnupg==0.5.3 # via ansible-sign @@ -225,7 +225,7 @@ rich==13.9.4 # ansible-lint # enrich # molecule -rpds-py==0.21.0 +rpds-py==0.22.1 # via # jsonschema # referencing