From 99cbc50707952cad999196b885ad681b287a4ce9 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:16:31 +0000 Subject: [PATCH 001/100] linting --- data_safe_haven/commands/sre.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/commands/sre.py b/data_safe_haven/commands/sre.py index 8c3e0b5cdc..2546d792c4 100644 --- a/data_safe_haven/commands/sre.py +++ b/data_safe_haven/commands/sre.py @@ -6,7 +6,10 @@ from data_safe_haven import console from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig -from data_safe_haven.exceptions import DataSafeHavenConfigError, DataSafeHavenError +from data_safe_haven.exceptions import ( + DataSafeHavenConfigError, + DataSafeHavenError, +) from data_safe_haven.external import AzureSdk, GraphApi from data_safe_haven.functions import current_ip_address, ip_address_in_list from data_safe_haven.infrastructure import SREProjectManager @@ -96,6 +99,7 @@ def deploy( ) # Set Entra options application = graph_api.get_application_by_name(context.entra_application_name) + if not application: msg = f"No Entra application '{context.entra_application_name}' was found. Please redeploy your SHM." raise DataSafeHavenConfigError(msg) From 4480f503d1086efb7b366c55741d074cffd2731e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 13 Nov 2024 12:16:54 +0000 Subject: [PATCH 002/100] Directly exit after credentials not confirmed --- data_safe_haven/external/api/credentials.py | 25 ++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index bfeb9c3aeb..30be974e9e 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -6,6 +6,7 @@ from typing import Any, ClassVar import jwt +import typer from azure.core.credentials import AccessToken, TokenCredential from azure.core.exceptions import ClientAuthenticationError from azure.identity import ( @@ -144,8 +145,7 @@ def get_credential(self) -> TokenCredential: self.logger.error( "Please authenticate with Azure: run '[green]az login[/]' using [bold]infrastructure administrator[/] credentials." ) - msg = "Error getting account information from Azure CLI." - raise DataSafeHavenAzureError(msg) from exc + raise typer.Exit(code=1) from exc return credential @@ -214,13 +214,18 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: raise DataSafeHavenAzureError(msg) from exc # Confirm that these are the desired credentials - self.confirm_credentials_interactive( - "Microsoft Graph API", - user_name=new_auth_record.username, - user_id=new_auth_record._home_account_id.split(".")[0], - tenant_name=new_auth_record._username.split("@")[1], - tenant_id=new_auth_record._tenant_id, - ) - + try: + self.confirm_credentials_interactive( + "Microsoft Graph API", + user_name=new_auth_record.username, + user_id=new_auth_record._home_account_id.split(".")[0], + tenant_name=new_auth_record._username.split("@")[1], + tenant_id=new_auth_record._tenant_id, + ) + except (CredentialUnavailableError, DataSafeHavenValueError) as exc: + self.logger.error( + "Please authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/]." + ) + raise typer.Exit(code=1) from exc # Return the credential return credential From 95619ecfbe459b04ecf82214f49a7b5a4592c2de Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:38:33 +0000 Subject: [PATCH 003/100] Check that a user belongs to the correct SHM domain when registering with SRE --- data_safe_haven/commands/users.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index fe413fa781..d90783f39b 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -146,11 +146,22 @@ def register( # List users users = UserHandler(context, graph_api) - available_usernames = users.get_usernames_entra_id() + # available_usernames = users.get_usernames_entra_id() + available_users = users.entra_users.list() + user_dict = {user.user_principal_name.split('@')[0]: user.user_principal_name.split('@')[1] for user in available_users} usernames_to_register = [] for username in usernames: - if username in available_usernames: - usernames_to_register.append(username) + if username in user_dict.keys(): + user_domain = user_dict[username] + if user_domain != shm_config.shm.fqdn: + logger.error( + f"Username '{username}' belongs to SHM domain '{user_domain}'.\n" + f"SRE '{sre_config.name}' is associated with SHM domain '{shm_config.shm.fqdn}'.\n" + "Users can only be registered to one SHM domain.\n" + "Please use 'dsh users add' to create a new user associated with the current SHM domain." + ) + else: + usernames_to_register.append(username) else: logger.error( f"Username '{username}' does not belong to this Data Safe Haven deployment." From 2943ab281959ce534a8f6171bfc23eb5d668b998 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:03:19 +0000 Subject: [PATCH 004/100] fix linting --- data_safe_haven/commands/users.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index d90783f39b..6c28096ded 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -148,7 +148,12 @@ def register( users = UserHandler(context, graph_api) # available_usernames = users.get_usernames_entra_id() available_users = users.entra_users.list() - user_dict = {user.user_principal_name.split('@')[0]: user.user_principal_name.split('@')[1] for user in available_users} + user_dict = { + user.user_principal_name.split("@")[0]: user.user_principal_name.split("@")[ + 1 + ] + for user in available_users + } usernames_to_register = [] for username in usernames: if username in user_dict.keys(): From fdcce90075480cc3835b8f44a542c3e5a1485550 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:48:06 +0000 Subject: [PATCH 005/100] get fqdn from stack output --- data_safe_haven/commands/users.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 6c28096ded..d479a1393e 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -10,6 +10,7 @@ from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi from data_safe_haven.logging import get_logger +from data_safe_haven.infrastructure import SREProjectManager users_command_group = typer.Typer() @@ -121,8 +122,9 @@ def register( try: shm_config = SHMConfig.from_remote(context) except DataSafeHavenError: - logger.error("Have you deployed the SHM?") - raise + msg = "Have you deployed the SHM?" + logger.error(msg) + raise DataSafeHavenError(msg) # Load Pulumi config pulumi_config = DSHPulumiConfig.from_remote(context) @@ -132,7 +134,13 @@ def register( if sre_config.name not in pulumi_config.project_names: msg = f"Could not load Pulumi settings for '{sre_config.name}'. Have you deployed the SRE?" logger.error(msg) - raise DataSafeHavenError(msg) + raise typer.Exit(1) + + sre_stack = SREProjectManager( + context=context, + config=sre_config, + pulumi_config=pulumi_config, + ) # Load GraphAPI graph_api = GraphApi.from_scopes( @@ -155,13 +163,14 @@ def register( for user in available_users } usernames_to_register = [] + shm_name = sre_stack.output("linked_shm")["name"] for username in usernames: if username in user_dict.keys(): user_domain = user_dict[username] - if user_domain != shm_config.shm.fqdn: + if shm_name not in user_domain: logger.error( f"Username '{username}' belongs to SHM domain '{user_domain}'.\n" - f"SRE '{sre_config.name}' is associated with SHM domain '{shm_config.shm.fqdn}'.\n" + f"SRE '{sre_config.name}' is associated with SHM domain '{shm_name}'.\n" "Users can only be registered to one SHM domain.\n" "Please use 'dsh users add' to create a new user associated with the current SHM domain." ) From 8b89c9c376e3f2c2a9df645e81c5f83a64bf16f6 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 11:49:34 +0000 Subject: [PATCH 006/100] Add exports from SRE stack --- data_safe_haven/infrastructure/programs/declarative_sre.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 15989bbe7b..9b0a5952ba 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -420,4 +420,6 @@ def __call__(self) -> None: pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) pulumi.export("remote_desktop", remote_desktop.exports) + pulumi.export("sre_fqdn", networking.sre_fqdn) + pulumi.export("linked_shm", self.context.name) pulumi.export("workspaces", workspaces.exports) From e9372021fafe911b9d8139dfef9b96b531a27bfa Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:05:55 +0000 Subject: [PATCH 007/100] Catch more specific exceptions rather than all exceptions --- data_safe_haven/external/api/graph_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/external/api/graph_api.py b/data_safe_haven/external/api/graph_api.py index 7d3b088672..e464061cf1 100644 --- a/data_safe_haven/external/api/graph_api.py +++ b/data_safe_haven/external/api/graph_api.py @@ -837,7 +837,7 @@ def read_applications(self) -> Sequence[dict[str, Any]]: "value" ] ] - except Exception as exc: + except (DataSafeHavenMicrosoftGraphError, requests.JSONDecodeError) as exc: msg = "Could not load list of applications." raise DataSafeHavenMicrosoftGraphError(msg) from exc From b2311d4f2f0318959281e3de982b30e60c7b4ff4 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 13:14:40 +0000 Subject: [PATCH 008/100] Tell user how to remove cached Graph API credential --- data_safe_haven/external/api/credentials.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index 30be974e9e..c8bdfa8236 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -224,7 +224,9 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: ) except (CredentialUnavailableError, DataSafeHavenValueError) as exc: self.logger.error( - "Please authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/]." + f"Delete the cached credential file [green]{authentication_record_path}[/] and\n" + "authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/].\n" + ) raise typer.Exit(code=1) from exc # Return the credential From 5149004ef4162e033b1be9fcac4e4b5eaa46ab00 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 15 Nov 2024 14:26:49 +0000 Subject: [PATCH 009/100] Add log analytics workspace to gitea and hedgedoc --- .../programs/declarative_sre.py | 29 ++++++++++--------- .../programs/sre/gitea_server.py | 9 ++++++ .../programs/sre/hedgedoc_server.py | 9 ++++++ .../programs/sre/user_services.py | 5 ++++ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 15989bbe7b..9563457806 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -308,6 +308,20 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy containerised user services user_services = SREUserServicesComponent( "sre_user_services", @@ -325,6 +339,7 @@ def __call__(self) -> None: ldap_username_attribute=ldap_username_attribute, ldap_user_search_base=ldap_user_search_base, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, nexus_admin_password=data.password_nexus_admin, resource_group_name=resource_group.name, software_packages=self.config.sre.software_packages, @@ -339,20 +354,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy desired state desired_state = SREDesiredStateComponent( "sre_desired_state", diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py index ab85ee51d8..2690de9c79 100644 --- a/data_safe_haven/infrastructure/programs/sre/gitea_server.py +++ b/data_safe_haven/infrastructure/programs/sre/gitea_server.py @@ -14,6 +14,7 @@ LocalDnsRecordProps, PostgresqlDatabaseComponent, PostgresqlDatabaseProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.utility import FileReader @@ -35,6 +36,7 @@ def __init__( ldap_user_filter: Input[str], ldap_user_search_base: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], sre_fqdn: Input[str], storage_account_key: Input[str], @@ -55,6 +57,7 @@ def __init__( self.ldap_user_filter = ldap_user_filter self.ldap_user_search_base = ldap_user_search_base self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.sre_fqdn = sre_fqdn self.storage_account_key = storage_account_key @@ -272,6 +275,12 @@ def __init__( ], ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), diff --git a/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py b/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py index 24cb858e68..d35efa81c5 100644 --- a/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py +++ b/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py @@ -15,6 +15,7 @@ LocalDnsRecordProps, PostgresqlDatabaseComponent, PostgresqlDatabaseProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.types import Ports @@ -37,6 +38,7 @@ def __init__( ldap_user_search_base: Input[str], ldap_username_attribute: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], sre_fqdn: Input[str], storage_account_key: Input[str], @@ -58,6 +60,7 @@ def __init__( self.ldap_user_search_base = ldap_user_search_base self.ldap_username_attribute = ldap_username_attribute self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.sre_fqdn = sre_fqdn self.storage_account_key = storage_account_key @@ -253,6 +256,12 @@ def __init__( ], ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 5eb04bdfbb..155169ff37 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -7,6 +7,7 @@ DockerHubCredentials, get_id_from_subnet, ) +from data_safe_haven.infrastructure.components import WrappedLogAnalyticsWorkspace from data_safe_haven.types import DatabaseSystem, SoftwarePackageCategory from .database_servers import SREDatabaseServerComponent, SREDatabaseServerProps @@ -35,6 +36,7 @@ def __init__( ldap_user_filter: Input[str], ldap_user_search_base: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], nexus_admin_password: Input[str], resource_group_name: Input[str], software_packages: SoftwarePackageCategory, @@ -58,6 +60,7 @@ def __init__( self.ldap_user_filter = ldap_user_filter self.ldap_user_search_base = ldap_user_search_base self.location = location + self.log_analytics_workspace = log_analytics_workspace self.nexus_admin_password = Output.secret(nexus_admin_password) self.resource_group_name = resource_group_name self.software_packages = software_packages @@ -109,6 +112,7 @@ def __init__( ldap_user_filter=props.ldap_user_filter, ldap_user_search_base=props.ldap_user_search_base, location=props.location, + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, sre_fqdn=props.sre_fqdn, storage_account_key=props.storage_account_key, @@ -134,6 +138,7 @@ def __init__( ldap_user_filter=props.ldap_user_filter, ldap_user_search_base=props.ldap_user_search_base, location=props.location, + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, sre_fqdn=props.sre_fqdn, storage_account_key=props.storage_account_key, From b792c45ed3d8b89648f62b8d6c3ad8303d1b7314 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:24:37 +0000 Subject: [PATCH 010/100] remove newline from error message --- data_safe_haven/external/api/credentials.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index c8bdfa8236..2dd5be6775 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -225,7 +225,7 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: except (CredentialUnavailableError, DataSafeHavenValueError) as exc: self.logger.error( f"Delete the cached credential file [green]{authentication_record_path}[/] and\n" - "authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/].\n" + "authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/]." ) raise typer.Exit(code=1) from exc From 041b5a4d31f88abda180d1eef1ae3a370245e53d Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:25:02 +0000 Subject: [PATCH 011/100] Catch another specific error type --- data_safe_haven/external/api/graph_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/external/api/graph_api.py b/data_safe_haven/external/api/graph_api.py index e464061cf1..85e3a872d4 100644 --- a/data_safe_haven/external/api/graph_api.py +++ b/data_safe_haven/external/api/graph_api.py @@ -13,6 +13,7 @@ from data_safe_haven import console from data_safe_haven.exceptions import ( + DataSafeHavenAzureError, DataSafeHavenMicrosoftGraphError, DataSafeHavenValueError, ) @@ -837,7 +838,7 @@ def read_applications(self) -> Sequence[dict[str, Any]]: "value" ] ] - except (DataSafeHavenMicrosoftGraphError, requests.JSONDecodeError) as exc: + except (DataSafeHavenAzureError, DataSafeHavenMicrosoftGraphError, requests.JSONDecodeError) as exc: msg = "Could not load list of applications." raise DataSafeHavenMicrosoftGraphError(msg) from exc From 7ae72ab495e4535f73f3224acf7e351cd0aa2520 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:25:17 +0000 Subject: [PATCH 012/100] Fix tests to reflect change in exception type --- tests/external/api/test_credentials.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/external/api/test_credentials.py b/tests/external/api/test_credentials.py index c0e631e912..f13588b316 100644 --- a/tests/external/api/test_credentials.py +++ b/tests/external/api/test_credentials.py @@ -4,8 +4,8 @@ DeviceCodeCredential, ) +from click.exceptions import Exit from data_safe_haven.directories import config_dir -from data_safe_haven.exceptions import DataSafeHavenAzureError from data_safe_haven.external.api.credentials import ( AzureSdkCredential, DeferredCredential, @@ -37,8 +37,7 @@ def test_confirm_credentials_interactive_fail( DeferredCredential.cache_ = set() credential = AzureSdkCredential(skip_confirmation=False) with pytest.raises( - DataSafeHavenAzureError, - match="Error getting account information from Azure CLI.", + Exit ): credential.get_credential() @@ -62,8 +61,7 @@ def test_decode_token_error( ): credential = AzureSdkCredential(skip_confirmation=True) with pytest.raises( - DataSafeHavenAzureError, - match="Error getting account information from Azure CLI.", + Exit ): credential.decode_token(credential.token) From ad95e2221ec0a2ed4307faf91afa33c5d8becf2b Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 15 Nov 2024 15:31:58 +0000 Subject: [PATCH 013/100] Add log analytics to apt proxy --- .../programs/declarative_sre.py | 29 ++++++++++--------- .../programs/sre/apt_proxy_server.py | 9 ++++++ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 9563457806..51c03ae7f8 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -209,6 +209,20 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy the apt proxy server apt_proxy_server = SREAptProxyServerComponent( "sre_apt_proxy_server", @@ -217,6 +231,7 @@ def __call__(self) -> None: containers_subnet=networking.subnet_apt_proxy_server, dns_server_ip=dns.ip_address, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group_name=resource_group.name, sre_fqdn=networking.sre_fqdn, storage_account_key=data.storage_account_data_configuration_key, @@ -308,20 +323,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy containerised user services user_services = SREUserServicesComponent( "sre_user_services", diff --git a/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py b/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py index ff1cb4b0da..d58a17a6de 100644 --- a/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py +++ b/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py @@ -12,6 +12,7 @@ FileShareFileProps, LocalDnsRecordComponent, LocalDnsRecordProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.types import PermittedDomains @@ -24,6 +25,7 @@ def __init__( containers_subnet: Input[str], dns_server_ip: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], sre_fqdn: Input[str], storage_account_key: Input[str], @@ -34,6 +36,7 @@ def __init__( ) self.dns_server_ip = dns_server_ip self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.sre_fqdn = sre_fqdn self.storage_account_key = storage_account_key @@ -119,6 +122,12 @@ def __init__( ], ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), From 7e6ff1dbac2a8e1ac853c82ff1bc568869d16a53 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 15 Nov 2024 15:34:03 +0000 Subject: [PATCH 014/100] Add log analytics to clamav mirror --- .../infrastructure/programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/clamav_mirror.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 51c03ae7f8..45e13b3616 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -248,6 +248,7 @@ def __call__(self) -> None: dns_server_ip=dns.ip_address, dockerhub_credentials=dockerhub_credentials, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group_name=resource_group.name, sre_fqdn=networking.sre_fqdn, storage_account_key=data.storage_account_data_configuration_key, diff --git a/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py b/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py index 203334a21b..e6f81df6cb 100644 --- a/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py +++ b/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py @@ -11,6 +11,7 @@ from data_safe_haven.infrastructure.components import ( LocalDnsRecordComponent, LocalDnsRecordProps, + WrappedLogAnalyticsWorkspace, ) @@ -22,6 +23,7 @@ def __init__( dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], sre_fqdn: Input[str], storage_account_key: Input[str], @@ -31,6 +33,7 @@ def __init__( self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.sre_fqdn = sre_fqdn self.storage_account_key = storage_account_key @@ -95,6 +98,12 @@ def __init__( ], ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), From f21323cb90b644820d67632dc1d09ffa611b263b Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 15 Nov 2024 15:36:37 +0000 Subject: [PATCH 015/100] Add log analytics to apricot container group --- .../infrastructure/programs/declarative_sre.py | 1 + data_safe_haven/infrastructure/programs/sre/identity.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 45e13b3616..c2a8ec8755 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -269,6 +269,7 @@ def __call__(self) -> None: entra_application_secret=entra.identity_application_secret, entra_tenant_id=shm_entra_tenant_id, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group_name=resource_group.name, shm_fqdn=shm_fqdn, sre_fqdn=networking.sre_fqdn, diff --git a/data_safe_haven/infrastructure/programs/sre/identity.py b/data_safe_haven/infrastructure/programs/sre/identity.py index 7839853384..4b06420190 100644 --- a/data_safe_haven/infrastructure/programs/sre/identity.py +++ b/data_safe_haven/infrastructure/programs/sre/identity.py @@ -13,6 +13,7 @@ from data_safe_haven.infrastructure.components import ( LocalDnsRecordComponent, LocalDnsRecordProps, + WrappedLogAnalyticsWorkspace, ) @@ -27,6 +28,7 @@ def __init__( entra_application_secret: Input[str], entra_tenant_id: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], shm_fqdn: Input[str], sre_fqdn: Input[str], @@ -40,6 +42,7 @@ def __init__( self.entra_application_secret = entra_application_secret self.entra_tenant_id = entra_tenant_id self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.shm_fqdn = shm_fqdn self.sre_fqdn = sre_fqdn @@ -163,6 +166,12 @@ def __init__( ], ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), From adbe0e73301cb644772ad40e8ac5c74cfe743bd8 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 15 Nov 2024 15:40:01 +0000 Subject: [PATCH 016/100] Add log analytics to gaucamole containers --- .../infrastructure/programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/remote_desktop.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index c2a8ec8755..d8f1f125ca 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -316,6 +316,7 @@ def __call__(self) -> None: ldap_user_filter=ldap_user_filter, ldap_user_search_base=ldap_user_search_base, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group_name=resource_group.name, storage_account_key=data.storage_account_data_configuration_key, storage_account_name=data.storage_account_data_configuration_name, diff --git a/data_safe_haven/infrastructure/programs/sre/remote_desktop.py b/data_safe_haven/infrastructure/programs/sre/remote_desktop.py index e2df83ede5..ba1e8b9816 100644 --- a/data_safe_haven/infrastructure/programs/sre/remote_desktop.py +++ b/data_safe_haven/infrastructure/programs/sre/remote_desktop.py @@ -15,6 +15,7 @@ FileShareFileProps, PostgresqlDatabaseComponent, PostgresqlDatabaseProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.utility import FileReader @@ -40,6 +41,7 @@ def __init__( ldap_user_filter: Input[str], ldap_user_search_base: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account_key: Input[str], storage_account_name: Input[str], @@ -65,6 +67,7 @@ def __init__( self.ldap_user_filter = ldap_user_filter self.ldap_user_search_base = ldap_user_search_base self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account_key = storage_account_key self.storage_account_name = storage_account_name @@ -348,6 +351,12 @@ def __init__( ), ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), From d51640b51032b49d35abd1e5f195c01d8e5a534a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 15 Nov 2024 15:44:03 +0000 Subject: [PATCH 017/100] Add log analytics to nexus container group --- .../infrastructure/programs/sre/software_repositories.py | 9 +++++++++ .../infrastructure/programs/sre/user_services.py | 1 + 2 files changed, 10 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/sre/software_repositories.py b/data_safe_haven/infrastructure/programs/sre/software_repositories.py index 013c9ffcdd..420ca5c5a2 100644 --- a/data_safe_haven/infrastructure/programs/sre/software_repositories.py +++ b/data_safe_haven/infrastructure/programs/sre/software_repositories.py @@ -14,6 +14,7 @@ FileShareFileProps, LocalDnsRecordComponent, LocalDnsRecordProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.types import Ports, SoftwarePackageCategory @@ -28,6 +29,7 @@ def __init__( dns_server_ip: Input[str], dockerhub_credentials: DockerHubCredentials, location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], nexus_admin_password: Input[str], resource_group_name: Input[str], software_packages: SoftwarePackageCategory, @@ -39,6 +41,7 @@ def __init__( self.dns_server_ip = dns_server_ip self.dockerhub_credentials = dockerhub_credentials self.location = location + self.log_analytics_workspace = log_analytics_workspace self.nexus_admin_password = Output.secret(nexus_admin_password) self.nexus_packages: str | None = { SoftwarePackageCategory.ANY: "all", @@ -250,6 +253,12 @@ def __init__( ], ), ], + diagnostics=containerinstance.ContainerGroupDiagnosticsArgs( + log_analytics=containerinstance.LogAnalyticsArgs( + workspace_id=props.log_analytics_workspace.workspace_id, + workspace_key=props.log_analytics_workspace.workspace_key, + ), + ), dns_config=containerinstance.DnsConfigurationArgs( name_servers=[props.dns_server_ip], ), diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py index 155169ff37..1418b3d11f 100644 --- a/data_safe_haven/infrastructure/programs/sre/user_services.py +++ b/data_safe_haven/infrastructure/programs/sre/user_services.py @@ -156,6 +156,7 @@ def __init__( dns_server_ip=props.dns_server_ip, dockerhub_credentials=props.dockerhub_credentials, location=props.location, + log_analytics_workspace=props.log_analytics_workspace, nexus_admin_password=props.nexus_admin_password, resource_group_name=props.resource_group_name, sre_fqdn=props.sre_fqdn, From 2b69d97ec9fd115854f97f028c95959a4c17e6dc Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:11:51 +0000 Subject: [PATCH 018/100] fix linting --- data_safe_haven/external/api/credentials.py | 1 - data_safe_haven/external/api/graph_api.py | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index 2dd5be6775..d9a7cc1c6c 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -226,7 +226,6 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: self.logger.error( f"Delete the cached credential file [green]{authentication_record_path}[/] and\n" "authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/]." - ) raise typer.Exit(code=1) from exc # Return the credential diff --git a/data_safe_haven/external/api/graph_api.py b/data_safe_haven/external/api/graph_api.py index 85e3a872d4..c118113cc2 100644 --- a/data_safe_haven/external/api/graph_api.py +++ b/data_safe_haven/external/api/graph_api.py @@ -838,7 +838,11 @@ def read_applications(self) -> Sequence[dict[str, Any]]: "value" ] ] - except (DataSafeHavenAzureError, DataSafeHavenMicrosoftGraphError, requests.JSONDecodeError) as exc: + except ( + DataSafeHavenAzureError, + DataSafeHavenMicrosoftGraphError, + requests.JSONDecodeError, + ) as exc: msg = "Could not load list of applications." raise DataSafeHavenMicrosoftGraphError(msg) from exc From 111db6fbdab2ccf051d7f595bf87521c98242692 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:12:01 +0000 Subject: [PATCH 019/100] fix linting --- tests/external/api/test_credentials.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/external/api/test_credentials.py b/tests/external/api/test_credentials.py index f13588b316..dcb7e10670 100644 --- a/tests/external/api/test_credentials.py +++ b/tests/external/api/test_credentials.py @@ -3,8 +3,8 @@ AzureCliCredential, DeviceCodeCredential, ) - from click.exceptions import Exit + from data_safe_haven.directories import config_dir from data_safe_haven.external.api.credentials import ( AzureSdkCredential, @@ -36,9 +36,7 @@ def test_confirm_credentials_interactive_fail( ): DeferredCredential.cache_ = set() credential = AzureSdkCredential(skip_confirmation=False) - with pytest.raises( - Exit - ): + with pytest.raises(Exit): credential.get_credential() def test_confirm_credentials_interactive_cache( @@ -60,9 +58,7 @@ def test_decode_token_error( self, mock_azureclicredential_get_token_invalid # noqa: ARG002 ): credential = AzureSdkCredential(skip_confirmation=True) - with pytest.raises( - Exit - ): + with pytest.raises(Exit): credential.decode_token(credential.token) From 52f86c170a2611beaf9116b2f0be35ebad38d970 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:28:58 +0000 Subject: [PATCH 020/100] Export sre_fqdn --- data_safe_haven/infrastructure/programs/declarative_sre.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 15989bbe7b..52f972cff7 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -420,4 +420,5 @@ def __call__(self) -> None: pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) pulumi.export("remote_desktop", remote_desktop.exports) + pulumi.export("sre_fqdn", networking.sre_fqdn) pulumi.export("workspaces", workspaces.exports) From bdaaf01c2818650604098232d888434f07e81efc Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Fri, 15 Nov 2024 16:42:30 +0000 Subject: [PATCH 021/100] Add final message displaying SRE FQDN in console --- data_safe_haven/commands/sre.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/data_safe_haven/commands/sre.py b/data_safe_haven/commands/sre.py index 8c3e0b5cdc..2ee6f3ec48 100644 --- a/data_safe_haven/commands/sre.py +++ b/data_safe_haven/commands/sre.py @@ -162,6 +162,12 @@ def deploy( timezone=sre_config.sre.timezone, ) manager.run() + + console.print( + f"Secure Research Environment '[green]{name}[/]' has been successfully deployed. \n" + f"The SRE can be accessed at https://{stack.output("sre_fqdn")}" + ) + except DataSafeHavenError as exc: logger.critical( f"Could not deploy Secure Research Environment '[green]{name}[/]'." From dad635c14c3a36db519d46ce186c18e2dca7e0fc Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 11:38:05 +0000 Subject: [PATCH 022/100] Break management docs into sections --- docs/source/management/data.md | 72 +++++++++++ docs/source/management/index.md | 216 +------------------------------- docs/source/management/sre.md | 51 ++++++++ docs/source/management/user.md | 84 +++++++++++++ 4 files changed, 213 insertions(+), 210 deletions(-) create mode 100644 docs/source/management/data.md create mode 100644 docs/source/management/sre.md create mode 100644 docs/source/management/user.md diff --git a/docs/source/management/data.md b/docs/source/management/data.md new file mode 100644 index 0000000000..988ff1cd11 --- /dev/null +++ b/docs/source/management/data.md @@ -0,0 +1,72 @@ +# Managing data ingress and egress + +## Data Ingress + +It is the {ref}`role_data_provider_representative`'s responsibility to upload the data required by the safe haven. + +The following steps show how to generate a temporary, write-only upload token that can be securely sent to the {ref}`role_data_provider_representative`, enabling them to upload the data: + +- In the Azure portal select **Subscriptions** then navigate to the subscription containing the relevant SHM +- Search for the resource group: `shm--sre--rg`, then click through to the storage account ending with `sensitivedata` +- Browse to **{menuselection}`Settings --> Networking`** and ensure that the data provider's IP address is one of those allowed under the **Firewall** header + - If it is not listed, modify and reupload the SRE configuration and redeploy the SRE using the `dsh` CLI, as per {ref}`deploy_sre` +- Browse to **{menuselection}`Data storage --> Containers`** from the menu on the left hand side +- Click **ingress** +- Browse to **{menuselection}`Settings --> Shared access tokens`** and do the following: + - Under **Signing method**, select **User delegation key** + - Under **Permissions**, check these boxes: + - **Write** + - **List** + - Set a 24 hour time window in the **Start and expiry date/time** (or an appropriate length of time) + - Leave everything else as default and click **{guilabel}`Generate SAS token and URL`** + - Copy the **Blob SAS URL** + + ```{image} ingress_token_write_only.png + :alt: write-only SAS token + :align: center + ``` + +- Send the **Blob SAS URL** to the data provider through a secure channel +- The data provider should now be able to upload data +- Validate successful data ingress + - Browse to **{menuselection}`Data storage --> Containers`** (in the middle of the page) + - Select the **ingress** container and ensure that the uploaded files are present + +## Data egress + +```{important} +Assessment of output must be completed **before** an egress link is created. +Outputs are potentially sensitive, and so an appropriate process must be applied to ensure that they are suitable for egress. +``` + +The {ref}`role_system_manager` creates a time-limited and IP restricted link to remove data from the environment. + +- In the Azure portal select **Subscriptions** then navigate to the subscription containing the relevant SHM +- Search for the resource group: `shm--sre--rg`, then click through to the storage account ending with `sensitivedata` +- Browse to **{menuselection}`Settings --> Networking`** and check the list of pre-approved IP addresses allowed under the **Firewall** header + - Ensure that the IP address of the person to receive the outputs is listed + - If it is not listed, modify and reupload the SRE configuration and redeploy the SRE using the `dsh` CLI, as per {ref}`deploy_sre` +- Browse to **{menuselection}`Data storage --> Containers`** +- Select the **egress** container +- Browse to **{menuselection}`Settings --> Shared access tokens`** and do the following: + - Under **Signing method**, select **User delegation key** + - Under **Permissions**, check these boxes: + - **Read** + - **List** + - Set a time window in the **Start and expiry date/time** that gives enough time for the person who will perform the secure egress download to do so + - Leave everything else as default and press **{guilabel}`Generate SAS token and URL`** + - Copy the **Blob SAS URL** + + ```{image} egress_token_read_only.png + :alt: Read-only SAS token + :align: center + ``` + +- Send the **Blob SAS URL** to the relevant person through a secure channel +- The appropriate person should now be able to download data + +## The output volume + +Once you have set up the egress connection in Azure Storage Explorer, you should be able to view data from the **output volume**, a read-write area intended for the extraction of results, such as figures for publication. +On the workspaces, this volume is `/mnt/output` and is shared between all workspaces in an SRE. +For more information on shared SRE storage volumes, consult the {ref}`Safe Haven User Guide `. diff --git a/docs/source/management/index.md b/docs/source/management/index.md index e9f49a5733..ae5312208a 100644 --- a/docs/source/management/index.md +++ b/docs/source/management/index.md @@ -1,215 +1,11 @@ # Management -## Managing users +:::{toctree} +:hidden: -### Add users to the Data Safe Haven - -:::{important} -You will need a full name, phone number, email address and country for each user. -::: - -1. You can add users directly in your Entra tenant, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-create-delete-users). - -2. Alternatively, you can add multiple users from a CSV file with columns named (`GivenName`, `Surname`, `Phone`, `Email`, `CountryCode`). - - (Optional) you can provide a `Domain` column if you like but this will otherwise default to the domain of your SHM - - {{warning}} **Phone** must be in [E.123 international format](https://en.wikipedia.org/wiki/E.123) - - {{warning}} **CountryCode** is the two letter [ISO 3166-1 Alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements) code for the country where the user is based - -::::{admonition} Example CSV user file -:class: dropdown tip - -:::{code} text -GivenName;Surname;Phone;Email;CountryCode -Ada;Lovelace;+44800456456;ada@lovelace.me;GB -Grace;Hopper;+18005550100;grace@nasa.gov;US -::: -:::: - -```{code} shell -$ dsh users add PATH_TO_MY_CSV_FILE -``` - -### List available users - -- You can do this from the [Microsoft Entra admin centre](https://entra.microsoft.com/) - - 1. Browse to **{menuselection}`Groups --> All Groups`** - 2. Click on the group named **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** - 3. Browse to **{menuselection}`Manage --> Members`** from the secondary menu on the left side - -- You can do this at the command line by running the following command: - - ```{code} shell - $ dsh users list YOUR_SRE_NAME - ``` - - which will give output like the following - - ``` - ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓ - ┃ username ┃ Entra ID ┃ SRE YOUR_SRE_NAME ┃ - ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩ - │ ada.lovelace │ x │ x │ - │ grace.hopper │ x │ x │ - │ ursula.franklin │ x │ │ - │ joan.clarke │ x │ │ - └──────────────────────────────┴──────────┴───────────────────┘ - ``` - -### Assign existing users to an SRE - -1. You can do this directly in your Entra tenant by adding them to the **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** group, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/groups-view-azure-portal#add-a-group-member). - -2. Alternatively, you can add multiple users from the command line: - - ```{code} shell - $ dsh users register YOUR_SRE_NAME -u USERNAME_1 -u USERNAME_2 - ``` - - where you must specify the usernames for each user you want to add to this SRE. - - :::{important} - Do not include the Entra ID domain part of the username, just the part before the @. - ::: - -### Manually register users for self-service password reset - -:::{tip} -Users created via the `dsh users` command line tool will be automatically registered for SSPR. +user.md +sre.md +data.md ::: -If you have manually created a user and want to enable SSPR, do the following - -- Go to the [Microsoft Entra admin centre](https://entra.microsoft.com/) -- Browse to **{menuselection}`Users --> All Users`** -- Select the user you want to enable SSPR for -- On the **{menuselection}`Manage --> Authentication Methods`** page fill out their contact info as follows: - - Ensure that you register **both** a phone number and an email address - - **Phone:** add the user's phone number with a space between the country code and the rest of the number (_e.g._ +44 7700900000) - - **Email:** enter the user's email address here - - Click the **{guilabel}`Save`** icon in the top panel - -## Managing SREs - -### List available SRE configurations and deployment status - -- Run the following if you want to check what SRE configurations are available in the current context, and whether those SREs are deployed - -```{code} shell -$ dsh config available -``` - -which will give output like the following - -```{code} shell -Available SRE configurations for context 'green': -┏━━━━━━━━━━━━━━┳━━━━━━━━━━┓ -┃ SRE Name ┃ Deployed ┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━━┩ -│ emerald │ x │ -│ jade │ │ -│ olive │ │ -└──────────────┴──────────┘ -``` - -### Remove a deployed Data Safe Haven - -- Run the following if you want to teardown a deployed SRE: - -```{code} shell -$ dsh sre teardown YOUR_SRE_NAME -``` - -::::{admonition} Tearing down an SRE is destructive and irreversible -:class: danger -Running `dsh sre teardown` will destroy **all** resources deployed within the SRE. -Ensure that any desired outputs have been extracted before deleting the SRE. -**All** data remaining on the SRE will be deleted. -The user groups for the SRE on Microsoft Entra ID will also be deleted. -:::: - -- Run the following if you want to teardown the deployed SHM: - -```{code} shell -$ dsh shm teardown -``` - -::::{admonition} Tearing down an SHM -:class: warning -Tearing down the SHM permanently deletes **all** remotely stored configuration and state data. -Tearing down the SHM also renders the SREs inaccessible to users and prevents them from being fully managed using the CLI. -All SREs associated with the SHM should be torn down before the SHM is torn down. -:::: - -## Managing data ingress and egress - -### Data Ingress - -It is the {ref}`role_data_provider_representative`'s responsibility to upload the data required by the safe haven. - -The following steps show how to generate a temporary, write-only upload token that can be securely sent to the {ref}`role_data_provider_representative`, enabling them to upload the data: - -- In the Azure portal select **Subscriptions** then navigate to the subscription containing the relevant SHM -- Search for the resource group: `shm--sre--rg`, then click through to the storage account ending with `sensitivedata` -- Browse to **{menuselection}`Settings --> Networking`** and ensure that the data provider's IP address is one of those allowed under the **Firewall** header - - If it is not listed, modify and reupload the SRE configuration and redeploy the SRE using the `dsh` CLI, as per {ref}`deploy_sre` -- Browse to **{menuselection}`Data storage --> Containers`** from the menu on the left hand side -- Click **ingress** -- Browse to **{menuselection}`Settings --> Shared access tokens`** and do the following: - - Under **Signing method**, select **User delegation key** - - Under **Permissions**, check these boxes: - - **Write** - - **List** - - Set a 24 hour time window in the **Start and expiry date/time** (or an appropriate length of time) - - Leave everything else as default and click **{guilabel}`Generate SAS token and URL`** - - Copy the **Blob SAS URL** - - ```{image} ingress_token_write_only.png - :alt: write-only SAS token - :align: center - ``` - -- Send the **Blob SAS URL** to the data provider through a secure channel -- The data provider should now be able to upload data -- Validate successful data ingress - - Browse to **{menuselection}`Data storage --> Containers`** (in the middle of the page) - - Select the **ingress** container and ensure that the uploaded files are present - -### Data egress - -```{important} -Assessment of output must be completed **before** an egress link is created. -Outputs are potentially sensitive, and so an appropriate process must be applied to ensure that they are suitable for egress. -``` - -The {ref}`role_system_manager` creates a time-limited and IP restricted link to remove data from the environment. - -- In the Azure portal select **Subscriptions** then navigate to the subscription containing the relevant SHM -- Search for the resource group: `shm--sre--rg`, then click through to the storage account ending with `sensitivedata` -- Browse to **{menuselection}`Settings --> Networking`** and check the list of pre-approved IP addresses allowed under the **Firewall** header - - Ensure that the IP address of the person to receive the outputs is listed - - If it is not listed, modify and reupload the SRE configuration and redeploy the SRE using the `dsh` CLI, as per {ref}`deploy_sre` -- Browse to **{menuselection}`Data storage --> Containers`** -- Select the **egress** container -- Browse to **{menuselection}`Settings --> Shared access tokens`** and do the following: - - Under **Signing method**, select **User delegation key** - - Under **Permissions**, check these boxes: - - **Read** - - **List** - - Set a time window in the **Start and expiry date/time** that gives enough time for the person who will perform the secure egress download to do so - - Leave everything else as default and press **{guilabel}`Generate SAS token and URL`** - - Copy the **Blob SAS URL** - - ```{image} egress_token_read_only.png - :alt: Read-only SAS token - :align: center - ``` - -- Send the **Blob SAS URL** to the relevant person through a secure channel -- The appropriate person should now be able to download data - -### The output volume - -Once you have set up the egress connection in Azure Storage Explorer, you should be able to view data from the **output volume**, a read-write area intended for the extraction of results, such as figures for publication. -On the workspaces, this volume is `/mnt/output` and is shared between all workspaces in an SRE. -For more information on shared SRE storage volumes, consult the {ref}`Safe Haven User Guide `. +Running a secure and productive Data Safe Haven requires a manager to conduct tasks which support users and to monitor the correct operation of the TRE. diff --git a/docs/source/management/sre.md b/docs/source/management/sre.md new file mode 100644 index 0000000000..284b179399 --- /dev/null +++ b/docs/source/management/sre.md @@ -0,0 +1,51 @@ +# Managing SREs + +## List available SRE configurations and deployment status + +- Run the following if you want to check what SRE configurations are available in the current context, and whether those SREs are deployed + +```{code} shell +$ dsh config available +``` + +which will give output like the following + +```{code} shell +Available SRE configurations for context 'green': +┏━━━━━━━━━━━━━━┳━━━━━━━━━━┓ +┃ SRE Name ┃ Deployed ┃ +┡━━━━━━━━━━━━━━╇━━━━━━━━━━┩ +│ emerald │ x │ +│ jade │ │ +│ olive │ │ +└──────────────┴──────────┘ +``` + +## Remove a deployed Data Safe Haven + +- Run the following if you want to teardown a deployed SRE: + +```{code} shell +$ dsh sre teardown YOUR_SRE_NAME +``` + +::::{admonition} Tearing down an SRE is destructive and irreversible +:class: danger +Running `dsh sre teardown` will destroy **all** resources deployed within the SRE. +Ensure that any desired outputs have been extracted before deleting the SRE. +**All** data remaining on the SRE will be deleted. +The user groups for the SRE on Microsoft Entra ID will also be deleted. +:::: + +- Run the following if you want to teardown the deployed SHM: + +```{code} shell +$ dsh shm teardown +``` + +::::{admonition} Tearing down an SHM +:class: warning +Tearing down the SHM permanently deletes **all** remotely stored configuration and state data. +Tearing down the SHM also renders the SREs inaccessible to users and prevents them from being fully managed using the CLI. +All SREs associated with the SHM should be torn down before the SHM is torn down. +:::: diff --git a/docs/source/management/user.md b/docs/source/management/user.md new file mode 100644 index 0000000000..d996321162 --- /dev/null +++ b/docs/source/management/user.md @@ -0,0 +1,84 @@ +# Managing users + +## Add users to the Data Safe Haven + +:::{important} +You will need a full name, phone number, email address and country for each user. +::: + +1. You can add users directly in your Entra tenant, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-create-delete-users). +1. Alternatively, you can add multiple users from a CSV file with columns named (`GivenName`, `Surname`, `Phone`, `Email`, `CountryCode`). + - (Optional) you can provide a `Domain` column if you like but this will otherwise default to the domain of your SHM + - {{warning}} **Phone** must be in [E.123 international format](https://en.wikipedia.org/wiki/E.123) + - {{warning}} **CountryCode** is the two letter [ISO 3166-1 Alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements) code for the country where the user is based + +::::{admonition} Example CSV user file +:class: dropdown tip + +:::{code} text +GivenName;Surname;Phone;Email;CountryCode +Ada;Lovelace;+44800456456;ada@lovelace.me;GB +Grace;Hopper;+18005550100;grace@nasa.gov;US +::: +:::: + +```{code} shell +$ dsh users add PATH_TO_MY_CSV_FILE +``` + +## List available users + +- You can do this from the [Microsoft Entra admin centre](https://entra.microsoft.com/) + 1. Browse to **{menuselection}`Groups --> All Groups`** + 1. Click on the group named **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** + 1. Browse to **{menuselection}`Manage --> Members`** from the secondary menu on the left side +- You can do this at the command line by running the following command: + + ```{code} shell + $ dsh users list YOUR_SRE_NAME + ``` + + which will give output like the following + + ``` + ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓ + ┃ username ┃ Entra ID ┃ SRE YOUR_SRE_NAME ┃ + ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩ + │ ada.lovelace │ x │ x │ + │ grace.hopper │ x │ x │ + │ ursula.franklin │ x │ │ + │ joan.clarke │ x │ │ + └──────────────────────────────┴──────────┴───────────────────┘ + ``` + +## Assign existing users to an SRE + +1. You can do this directly in your Entra tenant by adding them to the **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** group, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/groups-view-azure-portal#add-a-group-member). +1. Alternatively, you can add multiple users from the command line: + + ```{code} shell + $ dsh users register YOUR_SRE_NAME -u USERNAME_1 -u USERNAME_2 + ``` + + where you must specify the usernames for each user you want to add to this SRE. + + :::{important} + Do not include the Entra ID domain part of the username, just the part before the @. + ::: + +## Manually register users for self-service password reset + +:::{tip} +Users created via the `dsh users` command line tool will be automatically registered for SSPR. +::: + +If you have manually created a user and want to enable SSPR, do the following + +- Go to the [Microsoft Entra admin centre](https://entra.microsoft.com/) +- Browse to **{menuselection}`Users --> All Users`** +- Select the user you want to enable SSPR for +- On the **{menuselection}`Manage --> Authentication Methods`** page fill out their contact info as follows: + - Ensure that you register **both** a phone number and an email address + - **Phone:** add the user's phone number with a space between the country code and the rest of the number (_e.g._ +44 7700900000) + - **Email:** enter the user's email address here + - Click the **{guilabel}`Save`** icon in the top panel From 8455ab554c50585cb0f2961857fc64b09dc80998 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 11:38:26 +0000 Subject: [PATCH 023/100] Change markdown linting rules --- .mdlstyle.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mdlstyle.rb b/.mdlstyle.rb index 7ca3c2af8b..80b6e14d8c 100644 --- a/.mdlstyle.rb +++ b/.mdlstyle.rb @@ -6,7 +6,7 @@ exclude_rule 'MD013' exclude_rule 'MD024' rule 'MD026', :punctuation => ".,;" -rule 'MD029', :style => :ordered +rule 'MD029', :style => :one exclude_rule 'MD033' exclude_rule 'MD034' exclude_rule 'MD041' # this conflicts with MyST target anchors From 273ebd371af31f6719936549986cf86b2ea6c04c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 11:38:43 +0000 Subject: [PATCH 024/100] Add docs:clean script --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e3cb46525e..009ee997f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,7 @@ features = ["docs"] [tool.hatch.envs.docs.scripts] build = "sphinx-build -M html docs/source/ docs/build/ --fail-on-warning" +clean = "rm -r docs/build" lint = "mdl --style .mdlstyle.rb ./docs/source" [tool.hatch.envs.lint] From efdd9f6d5570c1c47c2700549b2cc08ef6eeed0a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 12:04:50 +0000 Subject: [PATCH 025/100] Fix cross-reference --- docs/source/deployment/security_checklist.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md index 2737b1cb5c..7c6036402a 100644 --- a/docs/source/deployment/security_checklist.md +++ b/docs/source/deployment/security_checklist.md @@ -50,7 +50,7 @@ In each SRE configuration ### Accounts -[Create a user account](../management/index.md#add-users-to-the-data-safe-haven) for the research user in your SHM. +[Create a user account](../management/user.md#add-users-to-the-data-safe-haven) for the research user in your SHM. Do not register this user with any SRE yet. ## 1. Multifactor authentication and password strength From 1a4de8911d05d278130564e7cecba55e07419739 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 12:05:22 +0000 Subject: [PATCH 026/100] Add docs for container logs --- docs/source/management/data.md | 2 +- docs/source/management/index.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/management/data.md b/docs/source/management/data.md index 988ff1cd11..9cacaf3806 100644 --- a/docs/source/management/data.md +++ b/docs/source/management/data.md @@ -1,6 +1,6 @@ # Managing data ingress and egress -## Data Ingress +## Data ingress It is the {ref}`role_data_provider_representative`'s responsibility to upload the data required by the safe haven. diff --git a/docs/source/management/index.md b/docs/source/management/index.md index ae5312208a..f8cd8ac0e0 100644 --- a/docs/source/management/index.md +++ b/docs/source/management/index.md @@ -6,6 +6,7 @@ user.md sre.md data.md +logs.md ::: Running a secure and productive Data Safe Haven requires a manager to conduct tasks which support users and to monitor the correct operation of the TRE. From fa114c155da04f22f15088710379dcf8590bca06 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 14:10:37 +0000 Subject: [PATCH 027/100] Fix unresolved merge conflict --- docs/source/management/index.md | 245 -------------------------------- 1 file changed, 245 deletions(-) diff --git a/docs/source/management/index.md b/docs/source/management/index.md index edde28b940..f8cd8ac0e0 100644 --- a/docs/source/management/index.md +++ b/docs/source/management/index.md @@ -9,249 +9,4 @@ data.md logs.md ::: -<<<<<<< HEAD Running a secure and productive Data Safe Haven requires a manager to conduct tasks which support users and to monitor the correct operation of the TRE. -======= -1. You can add users directly in your Entra tenant, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/how-to-create-delete-users). - -2. Alternatively, you can add multiple users from a CSV file with columns named (`GivenName`, `Surname`, `Phone`, `Email`, `CountryCode`). - - (Optional) you can provide a `Domain` column if you like but this will otherwise default to the domain of your SHM - - {{warning}} **Phone** must be in [E.123 international format](https://en.wikipedia.org/wiki/E.123) - - {{warning}} **CountryCode** is the two letter [ISO 3166-1 Alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2#Officially_assigned_code_elements) code for the country where the user is based - -::::{admonition} Example CSV user file -:class: dropdown tip - -:::{code} text -GivenName;Surname;Phone;Email;CountryCode -Ada;Lovelace;+44800456456;ada@lovelace.me;GB -Grace;Hopper;+18005550100;grace@nasa.gov;US -::: -:::: - -```{code} shell -$ dsh users add PATH_TO_MY_CSV_FILE -``` - -### List available users - -- You can do this from the [Microsoft Entra admin centre](https://entra.microsoft.com/) - - 1. Browse to **{menuselection}`Groups --> All Groups`** - 2. Click on the group named **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** - 3. Browse to **{menuselection}`Manage --> Members`** from the secondary menu on the left side - -- You can do this at the command line by running the following command: - - ```{code} shell - $ dsh users list YOUR_SRE_NAME - ``` - - which will give output like the following - - ``` - ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓ - ┃ username ┃ Entra ID ┃ SRE YOUR_SRE_NAME ┃ - ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩ - │ ada.lovelace │ x │ x │ - │ grace.hopper │ x │ x │ - │ ursula.franklin │ x │ │ - │ joan.clarke │ x │ │ - └──────────────────────────────┴──────────┴───────────────────┘ - ``` - -### Assign existing users to an SRE - -1. You can do this directly in your Entra tenant by adding them to the **Data Safe Haven SRE _YOUR\_SRE\_NAME_ Users** group, following the instructions [here](https://learn.microsoft.com/en-us/entra/fundamentals/groups-view-azure-portal#add-a-group-member). - -2. Alternatively, you can add multiple users from the command line: - - ```{code} shell - $ dsh users register YOUR_SRE_NAME -u USERNAME_1 -u USERNAME_2 - ``` - - where you must specify the usernames for each user you want to add to this SRE. - - :::{important} - Do not include the Entra ID domain part of the username, just the part before the @. - ::: - -### Manually register users for self-service password reset - -:::{tip} -Users created via the `dsh users` command line tool will be automatically registered for SSPR. -::: - -If you have manually created a user and want to enable SSPR, do the following - -- Go to the [Microsoft Entra admin centre](https://entra.microsoft.com/) -- Browse to **{menuselection}`Users --> All Users`** -- Select the user you want to enable SSPR for -- On the **{menuselection}`Manage --> Authentication Methods`** page fill out their contact info as follows: - - Ensure that you register **both** a phone number and an email address - - **Phone:** add the user's phone number with a space between the country code and the rest of the number (_e.g._ +44 7700900000) - - **Email:** enter the user's email address here - - Click the **{guilabel}`Save`** icon in the top panel - -## Managing SREs - -### List available SRE configurations and deployment status - -- Run the following if you want to check what SRE configurations are available in the current context, and whether those SREs are deployed - -```{code} shell -$ dsh config available -``` - -which will give output like the following - -```{code} shell -Available SRE configurations for context 'green': -┏━━━━━━━━━━━━━━┳━━━━━━━━━━┓ -┃ SRE Name ┃ Deployed ┃ -┡━━━━━━━━━━━━━━╇━━━━━━━━━━┩ -│ emerald │ x │ -│ jade │ │ -│ olive │ │ -└──────────────┴──────────┘ -``` - -### Remove a deployed Data Safe Haven - -- Run the following if you want to teardown a deployed SRE: - -```{code} shell -$ dsh sre teardown YOUR_SRE_NAME -``` - -::::{admonition} Tearing down an SRE is destructive and irreversible -:class: danger -Running `dsh sre teardown` will destroy **all** resources deployed within the SRE. -Ensure that any desired outputs have been extracted before deleting the SRE. -**All** data remaining on the SRE will be deleted. -The user groups for the SRE on Microsoft Entra ID will also be deleted. -:::: - -- Run the following if you want to teardown the deployed SHM: - -```{code} shell -$ dsh shm teardown -``` - -::::{admonition} Tearing down an SHM -:class: warning -Tearing down the SHM permanently deletes **all** remotely stored configuration and state data. -Tearing down the SHM also renders the SREs inaccessible to users and prevents them from being fully managed using the CLI. -All SREs associated with the SHM should be torn down before the SHM is torn down. -:::: - -### Updating SREs - -SREs are modified by updating the configuration then running the deploy command. - -- The existing configuration for the SRE can be shown using the following: - -```{code} shell -$ dsh config show YOUR_SRE_NAME -``` - -- If you do not have a local copy, you can write one with the `--file` option: - -```{code} shell -$ dsh config show YOUR_SRE_NAME --file YOUR_SRE_NAME.yaml -``` - -- Edit the configuration file locally, and upload the new version: - -```{code} shell -$ dsh config upload YOUR_SRE_NAME.yaml -``` - -- You will be shown the differences between the existing configuration and the new configuration and asked to confirm that they are correct. -- Finally, deploy your SRE to apply any changes: - -```{code} shell -$ dsh sre deploy YOUR_SRE_NAME -``` - -::::{admonition} Changing administrator IP addresses -:class: warning -The administrator IP addresses declared in the SRE configuration are used to create access rules for SRE infrastructure. -Therefore, after an SRE has been deployed, some changes can only be made from IP addresses on that list. - -As a consequence, if you want to update the list of administrator IP addresses, for example to add a new administrator, you must do so from an IP address that is already allowed. -:::: - -## Managing data ingress and egress - -### Data Ingress - -It is the {ref}`role_data_provider_representative`'s responsibility to upload the data required by the safe haven. - -The following steps show how to generate a temporary, write-only upload token that can be securely sent to the {ref}`role_data_provider_representative`, enabling them to upload the data: - -- In the Azure portal select **Subscriptions** then navigate to the subscription containing the relevant SHM -- Search for the resource group: `shm--sre--rg`, then click through to the storage account ending with `sensitivedata` -- Browse to **{menuselection}`Settings --> Networking`** and ensure that the data provider's IP address is one of those allowed under the **Firewall** header - - If it is not listed, modify and reupload the SRE configuration and redeploy the SRE using the `dsh` CLI, as per {ref}`deploy_sre` -- Browse to **{menuselection}`Data storage --> Containers`** from the menu on the left hand side -- Click **ingress** -- Browse to **{menuselection}`Settings --> Shared access tokens`** and do the following: - - Under **Signing method**, select **User delegation key** - - Under **Permissions**, check these boxes: - - **Write** - - **List** - - Set a 24 hour time window in the **Start and expiry date/time** (or an appropriate length of time) - - Leave everything else as default and click **{guilabel}`Generate SAS token and URL`** - - Copy the **Blob SAS URL** - - ```{image} ingress_token_write_only.png - :alt: write-only SAS token - :align: center - ``` - -- Send the **Blob SAS URL** to the data provider through a secure channel -- The data provider should now be able to upload data -- Validate successful data ingress - - Browse to **{menuselection}`Data storage --> Containers`** (in the middle of the page) - - Select the **ingress** container and ensure that the uploaded files are present - -### Data egress - -```{important} -Assessment of output must be completed **before** an egress link is created. -Outputs are potentially sensitive, and so an appropriate process must be applied to ensure that they are suitable for egress. -``` - -The {ref}`role_system_manager` creates a time-limited and IP restricted link to remove data from the environment. - -- In the Azure portal select **Subscriptions** then navigate to the subscription containing the relevant SHM -- Search for the resource group: `shm--sre--rg`, then click through to the storage account ending with `sensitivedata` -- Browse to **{menuselection}`Settings --> Networking`** and check the list of pre-approved IP addresses allowed under the **Firewall** header - - Ensure that the IP address of the person to receive the outputs is listed - - If it is not listed, modify and reupload the SRE configuration and redeploy the SRE using the `dsh` CLI, as per {ref}`deploy_sre` -- Browse to **{menuselection}`Data storage --> Containers`** -- Select the **egress** container -- Browse to **{menuselection}`Settings --> Shared access tokens`** and do the following: - - Under **Signing method**, select **User delegation key** - - Under **Permissions**, check these boxes: - - **Read** - - **List** - - Set a time window in the **Start and expiry date/time** that gives enough time for the person who will perform the secure egress download to do so - - Leave everything else as default and press **{guilabel}`Generate SAS token and URL`** - - Copy the **Blob SAS URL** - - ```{image} egress_token_read_only.png - :alt: Read-only SAS token - :align: center - ``` - -- Send the **Blob SAS URL** to the relevant person through a secure channel -- The appropriate person should now be able to download data - -### The output volume - -Once you have set up the egress connection in Azure Storage Explorer, you should be able to view data from the **output volume**, a read-write area intended for the extraction of results, such as figures for publication. -On the workspaces, this volume is `/mnt/output` and is shared between all workspaces in an SRE. -For more information on shared SRE storage volumes, consult the {ref}`Safe Haven User Guide `. ->>>>>>> origin/develop From 90f914cbc0bc05ab17cbf7975650c3d0f8a9b371 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 14:17:36 +0000 Subject: [PATCH 028/100] =?UTF-8?q?Add=20missing=20file=20=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/management/logs.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/source/management/logs.md diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md new file mode 100644 index 0000000000..f9a9948453 --- /dev/null +++ b/docs/source/management/logs.md @@ -0,0 +1,31 @@ +# Monitoring logs + +Logs are collected for numerous parts of a Data Safe Haven. +Some of these logs are ingested into a central location, an Azure [Log Analytics Workspace](https://learn.microsoft.com/azure/azure-monitor/logs/log-analytics-workspace-overview), and others are stored separately. + +## Log workspace + +Each SRE has its own Log Analytics Workspace. +You can view the workspaces by going to the Azure portal and navigating to [Log Analytics Workspaces](https://portal.azure.com/#browse/Microsoft.OperationalInsights%2Fworkspaces). +Select which log workspace you want to view by clicking on the workspace named `shm--sre--log`. + +The logs can be filtered using [Kusto Query Language (KQL)](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview). + +## Container logs + +Some of the Data Safe Haven infrastructure is provisioned as containers. +These include, + +- remote desktop portal +- package proxy +- Gitea and Hedgedoc + +Logs from all containers are ingested into the [SREs log workspace](#log-workspace). +There are two logs + +`ContainerEvents_CL` +: Event logs for the container instance resources such as starting, stopping, crashes and pulling images. + +`ContainerInstanceLog_CL` +: Container process logs. +: This is where you can view the output of the containerised applications and will be useful for debugging problems. From e6899229eba82a8e2c7a5d6051514cd7d27bff85 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 15:45:40 +0000 Subject: [PATCH 029/100] Revert some changes --- data_safe_haven/external/api/credentials.py | 26 ++++++++------------- tests/external/api/test_credentials.py | 12 +++++++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index d9a7cc1c6c..bfeb9c3aeb 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -6,7 +6,6 @@ from typing import Any, ClassVar import jwt -import typer from azure.core.credentials import AccessToken, TokenCredential from azure.core.exceptions import ClientAuthenticationError from azure.identity import ( @@ -145,7 +144,8 @@ def get_credential(self) -> TokenCredential: self.logger.error( "Please authenticate with Azure: run '[green]az login[/]' using [bold]infrastructure administrator[/] credentials." ) - raise typer.Exit(code=1) from exc + msg = "Error getting account information from Azure CLI." + raise DataSafeHavenAzureError(msg) from exc return credential @@ -214,19 +214,13 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: raise DataSafeHavenAzureError(msg) from exc # Confirm that these are the desired credentials - try: - self.confirm_credentials_interactive( - "Microsoft Graph API", - user_name=new_auth_record.username, - user_id=new_auth_record._home_account_id.split(".")[0], - tenant_name=new_auth_record._username.split("@")[1], - tenant_id=new_auth_record._tenant_id, - ) - except (CredentialUnavailableError, DataSafeHavenValueError) as exc: - self.logger.error( - f"Delete the cached credential file [green]{authentication_record_path}[/] and\n" - "authenticate with Graph API using [bold]global administrator credentials[/] for your [blue]Entra ID directory[/]." - ) - raise typer.Exit(code=1) from exc + self.confirm_credentials_interactive( + "Microsoft Graph API", + user_name=new_auth_record.username, + user_id=new_auth_record._home_account_id.split(".")[0], + tenant_name=new_auth_record._username.split("@")[1], + tenant_id=new_auth_record._tenant_id, + ) + # Return the credential return credential diff --git a/tests/external/api/test_credentials.py b/tests/external/api/test_credentials.py index dcb7e10670..c0e631e912 100644 --- a/tests/external/api/test_credentials.py +++ b/tests/external/api/test_credentials.py @@ -3,9 +3,9 @@ AzureCliCredential, DeviceCodeCredential, ) -from click.exceptions import Exit from data_safe_haven.directories import config_dir +from data_safe_haven.exceptions import DataSafeHavenAzureError from data_safe_haven.external.api.credentials import ( AzureSdkCredential, DeferredCredential, @@ -36,7 +36,10 @@ def test_confirm_credentials_interactive_fail( ): DeferredCredential.cache_ = set() credential = AzureSdkCredential(skip_confirmation=False) - with pytest.raises(Exit): + with pytest.raises( + DataSafeHavenAzureError, + match="Error getting account information from Azure CLI.", + ): credential.get_credential() def test_confirm_credentials_interactive_cache( @@ -58,7 +61,10 @@ def test_decode_token_error( self, mock_azureclicredential_get_token_invalid # noqa: ARG002 ): credential = AzureSdkCredential(skip_confirmation=True) - with pytest.raises(Exit): + with pytest.raises( + DataSafeHavenAzureError, + match="Error getting account information from Azure CLI.", + ): credential.decode_token(credential.token) From cdaefd33420296b1dee3b27b88e5e9f26eab6acc Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 15:46:13 +0000 Subject: [PATCH 030/100] Adjust formatting --- data_safe_haven/commands/sre.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/data_safe_haven/commands/sre.py b/data_safe_haven/commands/sre.py index 2546d792c4..29463b23b2 100644 --- a/data_safe_haven/commands/sre.py +++ b/data_safe_haven/commands/sre.py @@ -6,10 +6,7 @@ from data_safe_haven import console from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig -from data_safe_haven.exceptions import ( - DataSafeHavenConfigError, - DataSafeHavenError, -) +from data_safe_haven.exceptions import DataSafeHavenConfigError, DataSafeHavenError from data_safe_haven.external import AzureSdk, GraphApi from data_safe_haven.functions import current_ip_address, ip_address_in_list from data_safe_haven.infrastructure import SREProjectManager From 962a1e8b59746e305399f8b8a3705f47f88f724a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 19 Nov 2024 15:46:54 +0000 Subject: [PATCH 031/100] Move target name to class var --- data_safe_haven/external/api/credentials.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index bfeb9c3aeb..a32d65cf52 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -28,6 +28,7 @@ class DeferredCredential(TokenCredential): tokens_: ClassVar[dict[str, AccessToken]] = {} cache_: ClassVar[set[tuple[str, str]]] = set() + name: ClassVar[str] = "Credential name" def __init__( self, @@ -66,7 +67,6 @@ def get_credential(self) -> TokenCredential: def confirm_credentials_interactive( self, - target_name: str, user_name: str, user_id: str, tenant_name: str, @@ -86,7 +86,7 @@ def confirm_credentials_interactive( if (user_id, tenant_id) in DeferredCredential.cache_: return DeferredCredential.cache_.add((user_id, tenant_id)) - self.logger.info(f"You are logged into the [blue]{target_name}[/] as:") + self.logger.info(f"You are logged into the [blue]{self.name}[/] as:") self.logger.info(f"\tuser: [green]{user_name}[/] ({user_id})") self.logger.info(f"\ttenant: [green]{tenant_name}[/] ({tenant_id})") if not console.confirm("Are these details correct?", default_to_yes=True): @@ -118,6 +118,7 @@ class AzureSdkCredential(DeferredCredential): Uses AzureCliCredential for authentication """ + name: ClassVar[str] = "Azure CLI" def __init__( self, @@ -134,7 +135,6 @@ def get_credential(self) -> TokenCredential: try: decoded = self.decode_token(credential.get_token(*self.scopes).token) self.confirm_credentials_interactive( - "Azure CLI", user_name=decoded["name"], user_id=decoded["oid"], tenant_name=decoded["upn"].split("@")[1], @@ -155,6 +155,7 @@ class GraphApiCredential(DeferredCredential): Uses DeviceCodeCredential for authentication """ + name: ClassVar[str] = "Microsoft Graph API" def __init__( self, @@ -215,7 +216,6 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: # Confirm that these are the desired credentials self.confirm_credentials_interactive( - "Microsoft Graph API", user_name=new_auth_record.username, user_id=new_auth_record._home_account_id.split(".")[0], tenant_name=new_auth_record._username.split("@")[1], From db7e59a45d6fa43468a1a3014409d9d44876f01f Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 10:00:15 +0000 Subject: [PATCH 032/100] Simplify traceback --- data_safe_haven/exceptions/__init__.py | 10 ++++ data_safe_haven/external/api/credentials.py | 52 +++++++++++++-------- data_safe_haven/external/api/graph_api.py | 2 - 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/data_safe_haven/exceptions/__init__.py b/data_safe_haven/exceptions/__init__.py index b22d70e693..a858cfaf6d 100644 --- a/data_safe_haven/exceptions/__init__.py +++ b/data_safe_haven/exceptions/__init__.py @@ -28,6 +28,16 @@ class DataSafeHavenAzureError(DataSafeHavenError): pass +class DataSafeHavenCachedCredentialError(DataSafeHavenError): + """ + Exception class for handling errors related to cached credentials. + + Raise this error when a cached credential is not the credential a user wants to use. + """ + + pass + + class DataSafeHavenAzureStorageError(DataSafeHavenAzureError): """ Exception class for handling errors when interacting with Azure Storage. diff --git a/data_safe_haven/external/api/credentials.py b/data_safe_haven/external/api/credentials.py index a32d65cf52..82e444cea5 100644 --- a/data_safe_haven/external/api/credentials.py +++ b/data_safe_haven/external/api/credentials.py @@ -18,7 +18,11 @@ from data_safe_haven import console from data_safe_haven.directories import config_dir -from data_safe_haven.exceptions import DataSafeHavenAzureError, DataSafeHavenValueError +from data_safe_haven.exceptions import ( + DataSafeHavenAzureError, + DataSafeHavenCachedCredentialError, + DataSafeHavenValueError, +) from data_safe_haven.logging import get_logger from data_safe_haven.types import AzureSdkCredentialScope @@ -71,27 +75,24 @@ def confirm_credentials_interactive( user_id: str, tenant_name: str, tenant_id: str, - ) -> None: + ) -> bool: """ Allow user to confirm that credentials are correct. Responses are cached so the user will only be prompted once per run. If 'skip_confirmation' is set, then no confirmation will be performed. - - Raises: - DataSafeHavenValueError: if the user indicates that the credentials are wrong """ if self.skip_confirmation: - return + return True if (user_id, tenant_id) in DeferredCredential.cache_: - return + return True + DeferredCredential.cache_.add((user_id, tenant_id)) self.logger.info(f"You are logged into the [blue]{self.name}[/] as:") self.logger.info(f"\tuser: [green]{user_name}[/] ({user_id})") self.logger.info(f"\ttenant: [green]{tenant_name}[/] ({tenant_id})") - if not console.confirm("Are these details correct?", default_to_yes=True): - msg = "Selected credentials are incorrect." - raise DataSafeHavenValueError(msg) + + return console.confirm("Are these details correct?", default_to_yes=True) def get_token( self, @@ -118,6 +119,7 @@ class AzureSdkCredential(DeferredCredential): Uses AzureCliCredential for authentication """ + name: ClassVar[str] = "Azure CLI" def __init__( @@ -134,18 +136,22 @@ def get_credential(self) -> TokenCredential: # Confirm that these are the desired credentials try: decoded = self.decode_token(credential.get_token(*self.scopes).token) - self.confirm_credentials_interactive( - user_name=decoded["name"], - user_id=decoded["oid"], - tenant_name=decoded["upn"].split("@")[1], - tenant_id=decoded["tid"], - ) except (CredentialUnavailableError, DataSafeHavenValueError) as exc: + msg = "Error getting account information from Azure CLI." + raise DataSafeHavenAzureError(msg) from exc + + if not self.confirm_credentials_interactive( + user_name=decoded["name"], + user_id=decoded["oid"], + tenant_name=decoded["upn"].split("@")[1], + tenant_id=decoded["tid"], + ): self.logger.error( "Please authenticate with Azure: run '[green]az login[/]' using [bold]infrastructure administrator[/] credentials." ) - msg = "Error getting account information from Azure CLI." - raise DataSafeHavenAzureError(msg) from exc + msg = "Selected credentials are incorrect." + raise DataSafeHavenCachedCredentialError(msg) + return credential @@ -155,6 +161,7 @@ class GraphApiCredential(DeferredCredential): Uses DeviceCodeCredential for authentication """ + name: ClassVar[str] = "Microsoft Graph API" def __init__( @@ -215,12 +222,17 @@ def callback(verification_uri: str, user_code: str, _: datetime) -> None: raise DataSafeHavenAzureError(msg) from exc # Confirm that these are the desired credentials - self.confirm_credentials_interactive( + if not self.confirm_credentials_interactive( user_name=new_auth_record.username, user_id=new_auth_record._home_account_id.split(".")[0], tenant_name=new_auth_record._username.split("@")[1], tenant_id=new_auth_record._tenant_id, - ) + ): + self.logger.error( + f"Delete the cached credential file [green]{authentication_record_path}[/] and rerun dsh to authenticate with {self.name}" + ) + msg = "Selected credentials are incorrect." + raise DataSafeHavenCachedCredentialError(msg) # Return the credential return credential diff --git a/data_safe_haven/external/api/graph_api.py b/data_safe_haven/external/api/graph_api.py index c118113cc2..d77e78120d 100644 --- a/data_safe_haven/external/api/graph_api.py +++ b/data_safe_haven/external/api/graph_api.py @@ -13,7 +13,6 @@ from data_safe_haven import console from data_safe_haven.exceptions import ( - DataSafeHavenAzureError, DataSafeHavenMicrosoftGraphError, DataSafeHavenValueError, ) @@ -839,7 +838,6 @@ def read_applications(self) -> Sequence[dict[str, Any]]: ] ] except ( - DataSafeHavenAzureError, DataSafeHavenMicrosoftGraphError, requests.JSONDecodeError, ) as exc: From 4c2ec31bd6e16c7553ee67757524a706dbdb8f8c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 10:28:02 +0000 Subject: [PATCH 033/100] Fix Azure credential test --- tests/external/api/test_credentials.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/external/api/test_credentials.py b/tests/external/api/test_credentials.py index c0e631e912..a855697aa3 100644 --- a/tests/external/api/test_credentials.py +++ b/tests/external/api/test_credentials.py @@ -5,7 +5,7 @@ ) from data_safe_haven.directories import config_dir -from data_safe_haven.exceptions import DataSafeHavenAzureError +from data_safe_haven.exceptions import DataSafeHavenAzureError, DataSafeHavenCachedCredentialError from data_safe_haven.external.api.credentials import ( AzureSdkCredential, DeferredCredential, @@ -13,7 +13,7 @@ ) -class TestDeferredCredential: +class TestAzureSdkCredential: def test_confirm_credentials_interactive( self, mock_confirm_yes, # noqa: ARG002 @@ -37,8 +37,8 @@ def test_confirm_credentials_interactive_fail( DeferredCredential.cache_ = set() credential = AzureSdkCredential(skip_confirmation=False) with pytest.raises( - DataSafeHavenAzureError, - match="Error getting account information from Azure CLI.", + DataSafeHavenCachedCredentialError, + match="Selected credentials are incorrect.", ): credential.get_credential() @@ -67,8 +67,6 @@ def test_decode_token_error( ): credential.decode_token(credential.token) - -class TestAzureSdkCredential: def test_get_credential(self, mock_azureclicredential_get_token): # noqa: ARG002 credential = AzureSdkCredential(skip_confirmation=True) assert isinstance(credential.get_credential(), AzureCliCredential) From 12432ab283cef8f61ec475a7611fc20d25972ff1 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 10:33:46 +0000 Subject: [PATCH 034/100] Add stdout check to test --- tests/external/api/test_credentials.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/external/api/test_credentials.py b/tests/external/api/test_credentials.py index a855697aa3..730fc5dac8 100644 --- a/tests/external/api/test_credentials.py +++ b/tests/external/api/test_credentials.py @@ -33,6 +33,7 @@ def test_confirm_credentials_interactive_fail( self, mock_confirm_no, # noqa: ARG002 mock_azureclicredential_get_token, # noqa: ARG002 + capsys, ): DeferredCredential.cache_ = set() credential = AzureSdkCredential(skip_confirmation=False) @@ -41,6 +42,8 @@ def test_confirm_credentials_interactive_fail( match="Selected credentials are incorrect.", ): credential.get_credential() + out, _ = capsys.readouterr() + assert "Please authenticate with Azure: run 'az login'" in out def test_confirm_credentials_interactive_cache( self, From b7209fe93464cc5e2e2294b85743d1656f6d2b5a Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 10:50:33 +0000 Subject: [PATCH 035/100] Move tests path outside of default args This makes the script more flexible, you can add options without having to specify the test dir. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9304f1bd32..88e19b847e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,7 +158,7 @@ pip-compile-constraint = "default" features = ["test"] [tool.hatch.envs.test.scripts] -test = "coverage run -m pytest {args: tests}" +test = "coverage run -m pytest {args:} ./tests" test-report = "coverage report {args:}" test-coverage = ["test", "test-report"] From 407b1c7574c971800d42379d13c322b9f118bec9 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 11:03:02 +0000 Subject: [PATCH 036/100] Fix no application test --- tests/commands/test_sre.py | 8 +++++++- tests/conftest.py | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_sre.py b/tests/commands/test_sre.py index a13518a878..ad4bb3e8d0 100644 --- a/tests/commands/test_sre.py +++ b/tests/commands/test_sre.py @@ -5,7 +5,7 @@ from data_safe_haven.commands.sre import sre_command_group from data_safe_haven.config import Context, ContextManager from data_safe_haven.exceptions import DataSafeHavenAzureError -from data_safe_haven.external import AzureSdk +from data_safe_haven.external import AzureSdk, GraphApi class TestDeploySRE: @@ -31,13 +31,19 @@ def test_no_application( self, caplog: LogCaptureFixture, runner: CliRunner, + mocker, mock_azuresdk_get_subscription_name, # noqa: ARG002 mock_contextmanager_assert_context, # noqa: ARG002 mock_ip_1_2_3_4, # noqa: ARG002 mock_pulumi_config_from_remote_or_create, # noqa: ARG002 mock_shm_config_from_remote, # noqa: ARG002 mock_sre_config_from_remote, # noqa: ARG002 + mock_graphapi_get_credential, # noqa: ARG002 ) -> None: + mocker.patch.object( + GraphApi, "get_application_by_name", return_value=None + ) + result = runner.invoke(sre_command_group, ["deploy", "sandbox"]) assert result.exit_code == 1 assert ( diff --git a/tests/conftest.py b/tests/conftest.py index 5a8ce42847..a18ad166d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ ) from data_safe_haven.exceptions import DataSafeHavenAzureError from data_safe_haven.external import AzureSdk, PulumiAccount -from data_safe_haven.external.api.credentials import AzureSdkCredential +from data_safe_haven.external.api.credentials import GraphApiCredential, AzureSdkCredential from data_safe_haven.infrastructure import SREProjectManager from data_safe_haven.infrastructure.project_manager import ProjectManager from data_safe_haven.logging import init_logging @@ -215,6 +215,19 @@ def mock_azuresdk_get_subscription_name(mocker): ) +@fixture +def mock_graphapi_get_credential(mocker): + class MockCredential(TokenCredential): + def get_token(*args, **kwargs): # noqa: ARG002 + return AccessToken("dummy-token", 0) + + mocker.patch.object( + GraphApiCredential, + "get_credential", + return_value=MockCredential(), + ) + + @fixture def mock_azuresdk_get_credential(mocker): class MockCredential(TokenCredential): From 4c599533fe4504738effb8ee58d382f50e82235d Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 11:07:59 +0000 Subject: [PATCH 037/100] Run lint:fmt --- tests/commands/test_sre.py | 4 +--- tests/conftest.py | 5 ++++- tests/external/api/test_credentials.py | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/commands/test_sre.py b/tests/commands/test_sre.py index ad4bb3e8d0..9d2f79d07c 100644 --- a/tests/commands/test_sre.py +++ b/tests/commands/test_sre.py @@ -40,9 +40,7 @@ def test_no_application( mock_sre_config_from_remote, # noqa: ARG002 mock_graphapi_get_credential, # noqa: ARG002 ) -> None: - mocker.patch.object( - GraphApi, "get_application_by_name", return_value=None - ) + mocker.patch.object(GraphApi, "get_application_by_name", return_value=None) result = runner.invoke(sre_command_group, ["deploy", "sandbox"]) assert result.exit_code == 1 diff --git a/tests/conftest.py b/tests/conftest.py index a18ad166d3..8734d39ba1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,10 @@ ) from data_safe_haven.exceptions import DataSafeHavenAzureError from data_safe_haven.external import AzureSdk, PulumiAccount -from data_safe_haven.external.api.credentials import GraphApiCredential, AzureSdkCredential +from data_safe_haven.external.api.credentials import ( + AzureSdkCredential, + GraphApiCredential, +) from data_safe_haven.infrastructure import SREProjectManager from data_safe_haven.infrastructure.project_manager import ProjectManager from data_safe_haven.logging import init_logging diff --git a/tests/external/api/test_credentials.py b/tests/external/api/test_credentials.py index 730fc5dac8..e57bdb324b 100644 --- a/tests/external/api/test_credentials.py +++ b/tests/external/api/test_credentials.py @@ -5,7 +5,10 @@ ) from data_safe_haven.directories import config_dir -from data_safe_haven.exceptions import DataSafeHavenAzureError, DataSafeHavenCachedCredentialError +from data_safe_haven.exceptions import ( + DataSafeHavenAzureError, + DataSafeHavenCachedCredentialError, +) from data_safe_haven.external.api.credentials import ( AzureSdkCredential, DeferredCredential, From 08ebec9906fd57322e10984a8bb48d567b79ad8c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Wed, 20 Nov 2024 14:16:32 +0000 Subject: [PATCH 038/100] Tidy print statement and add colour --- data_safe_haven/commands/sre.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/commands/sre.py b/data_safe_haven/commands/sre.py index 4aa6c1e84c..14330c2cff 100644 --- a/data_safe_haven/commands/sre.py +++ b/data_safe_haven/commands/sre.py @@ -165,8 +165,9 @@ def deploy( manager.run() console.print( - f"Secure Research Environment '[green]{name}[/]' has been successfully deployed. \n" - f"The SRE can be accessed at https://{stack.output("sre_fqdn")}" + f"Secure Research Environment '[green]{name}[/]' has been successfully deployed.", + f"The SRE can be accessed at [green]https://{stack.output('sre_fqdn')}[/]", + sep="\n", ) except DataSafeHavenError as exc: From ca7483d87a3c468dc40b25d4ab5efa081a754edf Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:20:16 +0000 Subject: [PATCH 039/100] Update release checklist Remove irrelevant names section, add release making steps --- .github/ISSUE_TEMPLATE/release_checklist.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index a25064faa1..ade631052e 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -44,16 +44,16 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo - [ ] Update supported versions in `SECURITY.md` - [ ] Update pen test results in `VERSIONING.md` -## :computer: Release information +### Making the release -- **Version number:** _ -- **SHM ID:** _ -- **T2 SRE ID:** _ -- **T3 SRE ID:** _ +- [ ] Merge release branch into `latest` +- [ ] Push tag in the format `v0.0.1` to the merge commit into `latest` +- [ ] Ensure docs for the latest version are built and deployed on ReadTheDocs +- [ ] Push a build to PyPI +- [ ] Announce release on communications channels ## :deciduous_tree: Deployment problems - From 147939cacab07de554612a30c0c317593050e351 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:21:53 +0000 Subject: [PATCH 040/100] Remove powershell from bug report template --- .github/ISSUE_TEMPLATE/deployment_bug_report.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/deployment_bug_report.md b/.github/ISSUE_TEMPLATE/deployment_bug_report.md index 6cf453cc13..fa569038c2 100644 --- a/.github/ISSUE_TEMPLATE/deployment_bug_report.md +++ b/.github/ISSUE_TEMPLATE/deployment_bug_report.md @@ -29,7 +29,6 @@ Before reporting a problem please check the following. Replace the empty checkbo List of packages From 8a9df7a166a21f41ec6aa66858375db29b9392b0 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:22:52 +0000 Subject: [PATCH 041/100] Remove unused scripts --- .github/scripts/update_azure_data_studio.py | 19 ------ .github/scripts/update_dbeaver_drivers.py | 72 --------------------- .github/scripts/update_rstudio.py | 21 ------ 3 files changed, 112 deletions(-) delete mode 100644 .github/scripts/update_azure_data_studio.py delete mode 100644 .github/scripts/update_dbeaver_drivers.py delete mode 100644 .github/scripts/update_rstudio.py diff --git a/.github/scripts/update_azure_data_studio.py b/.github/scripts/update_azure_data_studio.py deleted file mode 100644 index 651e85fdfc..0000000000 --- a/.github/scripts/update_azure_data_studio.py +++ /dev/null @@ -1,19 +0,0 @@ -#! /usr/bin/env python3 -from lxml import html -import hashlib -import requests - -remote_page = requests.get("https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio", allow_redirects=True) -root = html.fromstring(remote_page.content) -short_link = root.xpath("//a[contains(text(), '.deb')]/@href")[0] - -remote_content = requests.get(short_link, allow_redirects=True) -sha256 = hashlib.sha256(remote_content.content).hexdigest() -version = remote_content.url.split("-")[-1].replace(".deb", "") -remote = "/".join(remote_content.url.split("/")[:-1] + ["|DEBFILE|"]) - -with open("deployment/secure_research_desktop/packages/deb-azuredatastudio.version", "w") as f_out: - f_out.write(f"hash: {sha256}\n") - f_out.write(f"version: {version}\n") - f_out.write("debfile: azuredatastudio-linux-|VERSION|.deb\n") - f_out.write(f"remote: {remote}\n") diff --git a/.github/scripts/update_dbeaver_drivers.py b/.github/scripts/update_dbeaver_drivers.py deleted file mode 100644 index 696a501858..0000000000 --- a/.github/scripts/update_dbeaver_drivers.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python3 -import json -from lxml import html -from natsort import natsorted -import requests - - -def get_latest_version(url, search_text): - """ - Get latest version number of a database driver from the Maven repository. - - Fetches the HTML page at the given URL, then converts it to an lxml tree. - Numeric strings are then extracted. - Note that mostly numeric strings for some drivers contain non-numeric text, - as different driver types exist for those drivers, even where the version number is the same. - The largest (latest) version number of the driver is then returned. - - Parameters - ---------- - url : str - The URL of the Maven repository containing the driver - search_text : str - Text to search for in the repository, to distinguish the driver from other files - - Returns - ------- - list - The latest available version number of the driver - """ - - remote_page = requests.get(url, allow_redirects=True) - root = html.fromstring(remote_page.content) - return natsorted([v for v in root.xpath("//a[contains(text(), '" + search_text + "')]/@href") if v != "../"])[-1].replace("/", "") - - -drivers = [ - { - 'name': "mssql_jdbc", - 'url': "https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/", - 'search_text': "jre8/" - }, - { - 'name': "pgjdbc", - 'url': "https://repo1.maven.org/maven2/org/postgresql/pgjdbc-versions/", - 'search_text': "/" - }, - { - 'name': "postgresql", - 'url': "https://repo1.maven.org/maven2/org/postgresql/postgresql/", - 'search_text': "/" - }, - { - 'name': "postgis_geometry", - 'url': "https://repo1.maven.org/maven2/net/postgis/postgis-geometry/", - 'search_text': "/" - }, - { - 'name': "postgis_jdbc", - 'url': "https://repo1.maven.org/maven2/net/postgis/postgis-jdbc/", - 'search_text': "/" - }, - { - 'name': "waffle_jna", - 'url': "https://repo1.maven.org/maven2/com/github/waffle/waffle-jna/", - 'search_text': "/" - } -] - -output = {driver['name']: get_latest_version(driver['url'], driver['search_text']) for driver in drivers} - -with open("deployment/secure_research_desktop/packages/dbeaver-driver-versions.json", "w") as f_out: - f_out.writelines(json.dumps(output, indent=4, sort_keys=True)) diff --git a/.github/scripts/update_rstudio.py b/.github/scripts/update_rstudio.py deleted file mode 100644 index ee36a35e66..0000000000 --- a/.github/scripts/update_rstudio.py +++ /dev/null @@ -1,21 +0,0 @@ -#! /usr/bin/env python3 -from lxml import html -import hashlib -import requests - -remote_page = requests.get("https://www.rstudio.com/products/rstudio/download/", allow_redirects=True) -root = html.fromstring(remote_page.content) -short_links = [link for link in root.xpath("//a[contains(text(), '.deb')]/@href") if "debian" not in link] - -for ubuntu_version in ["focal", "jammy"]: - short_link = [link for link in short_links if ubuntu_version in link][0] - remote_content = requests.get(short_link, allow_redirects=True) - sha256 = hashlib.sha256(remote_content.content).hexdigest() - version = "-".join(remote_content.url.split("/")[-1].split("-")[1:-1]) - remote = "/".join(remote_content.url.split("/")[:-1] + ["|DEBFILE|"]) - - with open(f"deployment/secure_research_desktop/packages/deb-rstudio-{ubuntu_version}.version", "w") as f_out: - f_out.write(f"hash: {sha256}\n") - f_out.write(f"version: {version}\n") - f_out.write("debfile: rstudio-|VERSION|-amd64.deb\n") - f_out.write(f"remote: {remote}\n") From d4aed3047975b0cb097528274948ea805d7b8cf9 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 22 Nov 2024 09:23:04 +0000 Subject: [PATCH 042/100] Remove security checklist template --- .github/security_checklist_template.md | 167 ------------------------- 1 file changed, 167 deletions(-) delete mode 100644 .github/security_checklist_template.md diff --git a/.github/security_checklist_template.md b/.github/security_checklist_template.md deleted file mode 100644 index b963331eef..0000000000 --- a/.github/security_checklist_template.md +++ /dev/null @@ -1,167 +0,0 @@ -# Security checklist -Running on SHM/SREs deployed using commit XXXXXXX - -## Summary -+ :white_check_mark: N tests passed -- :partly_sunny: N tests partially passed (see below for more details) -- :fast_forward: N tests skipped (see below for more details) -- :x: N tests failed (see below for more details) - -## Details -Some security checks were skipped since: -- No managed device was available -- No access to a physical space with its own dedicated network was possible - -### Multifactor Authentication and Password strength -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the SRE standard user cannot access the apps - +
:camera: Verify before adding to group: Microsoft Remote Desktop: Login works but apps cannot be viewed - -
- +
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA - -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that adding the **SRE standard user** to the SRE group on the domain controller does not give them access - +
:camera: Verify after adding to group: Microsoft Remote Desktop: Login works and apps can be viewed - -
- +
:camera: Verify after adding to group: Microsoft Remote Desktop: attempt to login to DSVM Main (Desktop) fails - -
- +
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA - -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** is able to successfully set up MFA - +
:camera: Verify: successfully set up MFA - -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can authenticate with MFA - +
:camera: Verify: Guacamole: respond to the MFA prompt - 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> -
- +
:camera: Verify: Microsoft Remote Desktop: attempt to log in to DSVM Main (Desktop) and respond to the MFA prompt - 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> -
- -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can access the DSVM desktop - +
:camera: Verify: Microsoft Remote Desktop: connect to DSVM Main (Desktop) - -
- +
:camera: Verify: Guacamole: connect to Desktop: Ubuntu0 - -
- -### Isolated Network -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connect to the SHM DC and NPS if connected to the SHM VPN -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the SHM DC and NPS if not connected to the SHM VPN -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from within a DSVM on the SRE network. - +
:camera: Verify: Connection fails - 122045859-8142bb00-cdde-11eb-920c-3a162a180647.png"> -
- +
:camera: Verify: that you cannot access a website using curl - -
- +
:camera: Verify: that you cannot get the IP address for a website using nslookup - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that users cannot connect between two SREs within the same SHM, even if they have access to both SREs - +
:camera: Verify: SSH connection fails - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules are set appropriately to block outgoing traffic - +
:camera: Verify: access rules - -
- -### User devices -#### Tier 2: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection - -#### Tier 3: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check user lacks root access -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection with an allow-listed IP address - -#### Tiers 2+: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses - +
:camera: Verify: access rules - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: All non-deployment NSGs have rules denying inbound connections from outside the Virtual Network - -### Physical security -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from outside was not tested -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from inside was not tested -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check the network IP ranges corresponding to the research spaces and compare against the IPs accepted by the firewall. -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so confirmation of physical measures was not tested - -### Remote connections - -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH - +
:camera: Verify: SSH connection by FQDN fails - -
- +
:camera: Verify: SSH connection by public IP address fails - -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: The remote desktop server is the only SRE resource with a public IP address - -### Copy-and-paste -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to paste local text into a DSVM -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to copy text from a DSVM -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Copy between VMs in an SRE succeeds - -### Data ingress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** secure upload token successfully created with write-only permissions -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** token was sent using a secure, out-of-band communication channel (e.g. secure email) -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an allow-listed IP address succeeds -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** downloading a file from an allow-listed IP address fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an non-allowed IP address fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection during lifetime of short-duration token succeeds -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection after lifetime of short-duration token fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading different file types succeeds - -### Storage volumes and egress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/output` volume -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can only read from the `/data` volume -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to their directory in `/home` -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/shared` volume -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can see the files ready for egress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can download egress-ready files - -### Software Ingress -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** expected software tools are installed - +
:camera: Verify: DBeaver, RStudio, PyCharm and Visual Studio Code available - 122056611-0a132400-cdea-11eb-9087-385ab296189e.png"> -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** secure upload token successfully created with write-only permissions -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading is possible only during the token lifetime -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** uploaded files are readable and can be installed on the DSVM -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** uploaded files are readable but cannot be installed on the DSVM - -### Package mirrors - -#### Tier 2: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages - +
:camera: Verify: botocore can be installed - -
- -#### Tier 3: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages - +
:camera: Verify: aero-calc can be installed; botocore cannot be installed - -
- -### Azure firewalls -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Admin has limited access to the internet - +
:camera: Verify: SHM DC cannot connect to google - 122067607-ff5d8c80-cdf3-11eb-8e20-a401faba0be4.png"> -
-+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Admin can download Windows updates - +
:camera: Verify: Windows updates can be downloaded - 122067641-071d3100-cdf4-11eb-9dc8-03938ff49e3a.png"> -
From fce9b8d8972517649b1639b8e8961d1d3227c6a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 03:14:54 +0000 Subject: [PATCH 043/100] Bump the production-dependencies group with 8 updates Bumps the production-dependencies group with 8 updates: | Package | From | To | | --- | --- | --- | | [pulumi-azure-native](https://github.com/pulumi/pulumi-azure-native) | `2.72.0` | `2.73.1` | | [pulumi](https://github.com/pulumi/pulumi) | `3.139.0` | `3.141.0` | | [pydantic](https://github.com/pydantic/pydantic) | `2.9.2` | `2.10.1` | | [typer](https://github.com/fastapi/typer) | `0.13.0` | `0.13.1` | | [ansible-dev-tools](https://github.com/ansible/ansible-dev-tools) | `24.10.2` | `24.11.0` | | [ansible](https://github.com/ansible-community/ansible-build-data) | `10.6.0` | `11.0.0` | | [ruff](https://github.com/astral-sh/ruff) | `0.7.4` | `0.8.0` | | [coverage](https://github.com/nedbat/coveragepy) | `7.6.7` | `7.6.8` | Updates `pulumi-azure-native` from 2.72.0 to 2.73.1 - [Release notes](https://github.com/pulumi/pulumi-azure-native/releases) - [Changelog](https://github.com/pulumi/pulumi-azure-native/blob/master/CHANGELOG_OLD.md) - [Commits](https://github.com/pulumi/pulumi-azure-native/compare/v2.72.0...v2.73.1) Updates `pulumi` from 3.139.0 to 3.141.0 - [Release notes](https://github.com/pulumi/pulumi/releases) - [Changelog](https://github.com/pulumi/pulumi/blob/master/CHANGELOG.md) - [Commits](https://github.com/pulumi/pulumi/compare/v3.139.0...v3.141.0) Updates `pydantic` from 2.9.2 to 2.10.1 - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.9.2...v2.10.1) Updates `typer` from 0.13.0 to 0.13.1 - [Release notes](https://github.com/fastapi/typer/releases) - [Changelog](https://github.com/fastapi/typer/blob/master/docs/release-notes.md) - [Commits](https://github.com/fastapi/typer/compare/0.13.0...0.13.1) Updates `ansible-dev-tools` from 24.10.2 to 24.11.0 - [Release notes](https://github.com/ansible/ansible-dev-tools/releases) - [Commits](https://github.com/ansible/ansible-dev-tools/compare/v24.10.2...v24.11.0) Updates `ansible` from 10.6.0 to 11.0.0 - [Changelog](https://github.com/ansible-community/ansible-build-data/blob/main/docs/release-process.md) - [Commits](https://github.com/ansible-community/ansible-build-data/compare/10.6.0...11.0.0) Updates `ruff` from 0.7.4 to 0.8.0 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.7.4...0.8.0) Updates `coverage` from 7.6.7 to 7.6.8 - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.6.7...7.6.8) --- updated-dependencies: - dependency-name: pulumi-azure-native dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pulumi dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: typer dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: ansible-dev-tools dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: ansible dependency-type: direct:production update-type: version-update:semver-major dependency-group: production-dependencies - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies ... Signed-off-by: dependabot[bot] --- pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6a1435e4eb..49099b230d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,17 +45,17 @@ dependencies = [ "cryptography==43.0.3", "fqdn==1.5.1", "psycopg[binary]==3.1.19", # needed for installation on older MacOS versions - "pulumi-azure-native==2.72.0", + "pulumi-azure-native==2.73.1", "pulumi-azuread==6.0.1", "pulumi-random==4.16.7", - "pulumi==3.139.0", - "pydantic==2.9.2", + "pulumi==3.141.0", + "pydantic==2.10.1", "pyjwt[crypto]==2.10.0", "pytz==2024.2", "pyyaml==6.0.2", "rich==13.9.4", "simple-acme-dns==3.2.0", - "typer==0.13.0", + "typer==0.13.1", "websocket-client==1.8.0", ] @@ -73,13 +73,13 @@ docs = [ "sphinx==8.1.3", ] lint = [ - "ansible-dev-tools==24.10.2", - "ansible==10.6.0", + "ansible-dev-tools==24.11.0", + "ansible==11.0.0", "black==24.10.0", "mypy==1.13.0", "pandas-stubs==2.2.3.241009", - "pydantic==2.9.2", - "ruff==0.7.4", + "pydantic==2.10.1", + "ruff==0.8.0", "types-appdirs==1.4.3.5", "types-chevron==0.14.2.20240310", "types-pytz==2024.2.0.20241003", @@ -87,7 +87,7 @@ lint = [ "types-requests==2.32.0.20241016", ] test = [ - "coverage==7.6.7", + "coverage==7.6.8", "freezegun==1.5.1", "pytest-mock==3.14.0", "pytest==8.3.3", From eaef834c0cb278a9c7b0810c27154bea7c9ee0dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 25 Nov 2024 03:21:58 +0000 Subject: [PATCH 044/100] [dependabot skip] :wrench: Update Python requirements files --- .hatch/requirements-docs.txt | 2 +- .hatch/requirements-lint.txt | 20 ++++++++++---------- .hatch/requirements-test.txt | 28 ++++++++++++++-------------- .hatch/requirements.txt | 22 +++++++++++----------- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.hatch/requirements-docs.txt b/.hatch/requirements-docs.txt index 95fcbdfd41..cebd1e3c16 100644 --- a/.hatch/requirements-docs.txt +++ b/.hatch/requirements-docs.txt @@ -91,7 +91,7 @@ typing-extensions==4.12.2 # via pydata-sphinx-theme urllib3==2.2.3 # via requests -wheel==0.45.0 +wheel==0.45.1 # via sphinx-togglebutton # The following packages are considered to be unsafe in a requirements file: diff --git a/.hatch/requirements-lint.txt b/.hatch/requirements-lint.txt index 295c694a65..e1ab89f54e 100644 --- a/.hatch/requirements-lint.txt +++ b/.hatch/requirements-lint.txt @@ -1,13 +1,13 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.12 # -# - ansible-dev-tools==24.10.2 -# - ansible==10.6.0 +# - ansible-dev-tools==24.11.0 +# - ansible==11.0.0 # - black==24.10.0 # - mypy==1.13.0 # - pandas-stubs==2.2.3.241009 -# - pydantic==2.9.2 -# - ruff==0.7.4 +# - pydantic==2.10.1 +# - ruff==0.8.0 # - types-appdirs==1.4.3.5 # - types-chevron==0.14.2.20240310 # - types-pytz==2024.2.0.20241003 @@ -17,7 +17,7 @@ annotated-types==0.7.0 # via pydantic -ansible==10.6.0 +ansible==11.0.0 # via hatch.envs.lint ansible-builder==3.1.0 # via @@ -29,7 +29,7 @@ ansible-compat==24.10.0 # ansible-lint # molecule # pytest-ansible -ansible-core==2.17.6 +ansible-core==2.18.0 # via # ansible # ansible-compat @@ -40,7 +40,7 @@ ansible-creator==24.11.0 # via ansible-dev-tools ansible-dev-environment==24.9.0 # via ansible-dev-tools -ansible-dev-tools==24.10.2 +ansible-dev-tools==24.11.0 # via hatch.envs.lint ansible-lint==24.10.0 # via @@ -178,9 +178,9 @@ ptyprocess==0.7.0 # via pexpect pycparser==2.22 # via cffi -pydantic==2.9.2 +pydantic==2.10.1 # via hatch.envs.lint -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via pydantic pygments==2.18.0 # via rich @@ -233,7 +233,7 @@ ruamel-yaml==0.18.6 # via ansible-lint ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.4 +ruff==0.8.0 # via hatch.envs.lint subprocess-tee==0.4.2 # via diff --git a/.hatch/requirements-test.txt b/.hatch/requirements-test.txt index 643331837b..8c95d7dce0 100644 --- a/.hatch/requirements-test.txt +++ b/.hatch/requirements-test.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.12 # -# [constraints] .hatch/requirements.txt (SHA256: ca6dfe8295dd8d2e6e4ade0fce58d158854ce5df89be8d092b36c34fe2679f3f) +# [constraints] .hatch/requirements.txt (SHA256: 3586aa93da255077aac182009c06aa28b96ec15387beec4148e3bebd2b9f8852) # # - appdirs==1.4.4 # - azure-core==1.32.0 @@ -24,19 +24,19 @@ # - cryptography==43.0.3 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.72.0 +# - pulumi-azure-native==2.73.1 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.139.0 -# - pydantic==2.9.2 +# - pulumi==3.141.0 +# - pydantic==2.10.1 # - pyjwt[crypto]==2.10.0 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.0 +# - typer==0.13.1 # - websocket-client==1.8.0 -# - coverage==7.6.7 +# - coverage==7.6.8 # - freezegun==1.5.1 # - pytest-mock==3.14.0 # - pytest==8.3.3 @@ -180,7 +180,7 @@ click==8.1.7 # via # -c .hatch/requirements.txt # typer -coverage==7.6.7 +coverage==7.6.8 # via hatch.envs.test cryptography==43.0.3 # via @@ -195,7 +195,7 @@ cryptography==43.0.3 # msal # pyjwt # pyopenssl -debugpy==1.8.8 +debugpy==1.8.9 # via # -c .hatch/requirements.txt # pulumi @@ -251,7 +251,7 @@ mdurl==0.1.2 # via # -c .hatch/requirements.txt # markdown-it-py -msal==1.31.0 +msal==1.31.1 # via # -c .hatch/requirements.txt # azure-identity @@ -295,14 +295,14 @@ psycopg-binary==3.1.19 # via # -c .hatch/requirements.txt # psycopg -pulumi==3.139.0 +pulumi==3.141.0 # via # -c .hatch/requirements.txt # hatch.envs.test # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.72.0 +pulumi-azure-native==2.73.1 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -318,11 +318,11 @@ pycparser==2.22 # via # -c .hatch/requirements.txt # cffi -pydantic==2.9.2 +pydantic==2.10.1 # via # -c .hatch/requirements.txt # hatch.envs.test -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via # -c .hatch/requirements.txt # pydantic @@ -403,7 +403,7 @@ six==1.16.0 # azure-core # pulumi # python-dateutil -typer==0.13.0 +typer==0.13.1 # via # -c .hatch/requirements.txt # hatch.envs.test diff --git a/.hatch/requirements.txt b/.hatch/requirements.txt index b0f7aff926..82ad061fc0 100644 --- a/.hatch/requirements.txt +++ b/.hatch/requirements.txt @@ -22,17 +22,17 @@ # - cryptography==43.0.3 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.72.0 +# - pulumi-azure-native==2.73.1 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.139.0 -# - pydantic==2.9.2 +# - pulumi==3.141.0 +# - pydantic==2.10.1 # - pyjwt[crypto]==2.10.0 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.0 +# - typer==0.13.1 # - websocket-client==1.8.0 # @@ -134,7 +134,7 @@ cryptography==43.0.3 # msal # pyjwt # pyopenssl -debugpy==1.8.8 +debugpy==1.8.9 # via pulumi dill==0.3.9 # via pulumi @@ -167,7 +167,7 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -msal==1.31.0 +msal==1.31.1 # via # azure-identity # msal-extensions @@ -192,13 +192,13 @@ psycopg==3.1.19 # via hatch.envs.default psycopg-binary==3.1.19 # via psycopg -pulumi==3.139.0 +pulumi==3.141.0 # via # hatch.envs.default # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.72.0 +pulumi-azure-native==2.73.1 # via hatch.envs.default pulumi-azuread==6.0.1 # via hatch.envs.default @@ -206,9 +206,9 @@ pulumi-random==4.16.7 # via hatch.envs.default pycparser==2.22 # via cffi -pydantic==2.9.2 +pydantic==2.10.1 # via hatch.envs.default -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via pydantic pygments==2.18.0 # via rich @@ -257,7 +257,7 @@ six==1.16.0 # via # azure-core # pulumi -typer==0.13.0 +typer==0.13.1 # via hatch.envs.default typing-extensions==4.12.2 # via From 07659ad436f79a43daacd842e54277ac71d5d524 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 25 Nov 2024 09:41:20 +0000 Subject: [PATCH 045/100] Fix linting --- data_safe_haven/console/__init__.py | 2 +- data_safe_haven/external/__init__.py | 2 +- data_safe_haven/infrastructure/common/__init__.py | 4 ++-- data_safe_haven/infrastructure/components/__init__.py | 2 +- data_safe_haven/infrastructure/components/wrapped/__init__.py | 2 +- data_safe_haven/types/__init__.py | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/data_safe_haven/console/__init__.py b/data_safe_haven/console/__init__.py index 133a48fc12..f30bda2882 100644 --- a/data_safe_haven/console/__init__.py +++ b/data_safe_haven/console/__init__.py @@ -1,5 +1,5 @@ from .format import tabulate -from .pretty import pretty_print as print # noqa: A001 +from .pretty import pretty_print as print # noqa: A004 from .prompts import confirm __all__ = [ diff --git a/data_safe_haven/external/__init__.py b/data_safe_haven/external/__init__.py index 5e46325958..d26ef75058 100644 --- a/data_safe_haven/external/__init__.py +++ b/data_safe_haven/external/__init__.py @@ -6,10 +6,10 @@ from .interface.pulumi_account import PulumiAccount __all__ = [ - "AzureSdk", "AzureContainerInstance", "AzureIPv4Range", "AzurePostgreSQLDatabase", + "AzureSdk", "GraphApi", "PulumiAccount", ] diff --git a/data_safe_haven/infrastructure/common/__init__.py b/data_safe_haven/infrastructure/common/__init__.py index 6106cac731..85184d6574 100644 --- a/data_safe_haven/infrastructure/common/__init__.py +++ b/data_safe_haven/infrastructure/common/__init__.py @@ -16,6 +16,8 @@ __all__ = [ "DockerHubCredentials", + "SREDnsIpRanges", + "SREIpRanges", "get_address_prefixes_from_subnet", "get_available_ips_from_subnet", "get_id_from_rg", @@ -27,6 +29,4 @@ "get_name_from_subnet", "get_name_from_vnet", "get_subscription_id_from_rg", - "SREDnsIpRanges", - "SREIpRanges", ] diff --git a/data_safe_haven/infrastructure/components/__init__.py b/data_safe_haven/infrastructure/components/__init__.py index 2b3dd67e7a..f4b93b9c3d 100644 --- a/data_safe_haven/infrastructure/components/__init__.py +++ b/data_safe_haven/infrastructure/components/__init__.py @@ -41,11 +41,11 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", - "WrappedNFSV3StorageAccount", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "SSLCertificate", "SSLCertificateProps", "VMComponent", "WrappedLogAnalyticsWorkspace", + "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/wrapped/__init__.py b/data_safe_haven/infrastructure/components/wrapped/__init__.py index ef6e7374d2..b449f46859 100644 --- a/data_safe_haven/infrastructure/components/wrapped/__init__.py +++ b/data_safe_haven/infrastructure/components/wrapped/__init__.py @@ -2,6 +2,6 @@ from .nfsv3_storage_account import WrappedNFSV3StorageAccount __all__ = [ - "WrappedNFSV3StorageAccount", "WrappedLogAnalyticsWorkspace", + "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/types/__init__.py b/data_safe_haven/types/__init__.py index 728df06c19..bfe1f6898a 100644 --- a/data_safe_haven/types/__init__.py +++ b/data_safe_haven/types/__init__.py @@ -34,14 +34,14 @@ "AzureDnsZoneNames", "AzureLocation", "AzurePremiumFileShareSize", - "AzureServiceTag", "AzureSdkCredentialScope", + "AzureServiceTag", "AzureSubscriptionName", "AzureVmSku", "DatabaseSystem", "EmailAddress", - "EntraApplicationId", "EntraAppPermissionType", + "EntraApplicationId", "EntraGroupName", "EntraSignInAudienceType", "FirewallPriorities", From 408b2d02fbc1a3c23c45295b0b479e0e2b449a45 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 25 Nov 2024 15:23:40 +0000 Subject: [PATCH 046/100] Add diagnostic settings for firewall --- .../programs/declarative_sre.py | 29 +++++++-------- .../infrastructure/programs/sre/firewall.py | 35 ++++++++++++++++++- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 78467f201b..f69fc9cd45 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -163,12 +163,27 @@ def __call__(self) -> None: ), ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy SRE firewall SREFirewallComponent( "sre_firewall", self.stack_name, SREFirewallProps( location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group_name=resource_group.name, route_table_name=networking.route_table_name, subnet_apt_proxy_server=networking.subnet_apt_proxy_server, @@ -209,20 +224,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy the apt proxy server apt_proxy_server = SREAptProxyServerComponent( "sre_apt_proxy_server", diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 97f7a885b7..1f46db980b 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -3,12 +3,13 @@ from collections.abc import Mapping from pulumi import ComponentResource, Input, Output, ResourceOptions -from pulumi_azure_native import network +from pulumi_azure_native import insights, network from data_safe_haven.infrastructure.common import ( get_address_prefixes_from_subnet, get_id_from_subnet, ) +from data_safe_haven.infrastructure.components import WrappedLogAnalyticsWorkspace from data_safe_haven.types import ( FirewallPriorities, ForbiddenDomains, @@ -23,6 +24,7 @@ class SREFirewallProps: def __init__( self, location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], route_table_name: Input[str], subnet_apt_proxy_server: Input[network.GetSubnetResult], @@ -35,6 +37,7 @@ def __init__( subnet_workspaces: Input[network.GetSubnetResult], ) -> None: self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.route_table_name = route_table_name self.subnet_apt_proxy_server_prefixes = Output.from_input( @@ -331,6 +334,36 @@ def __init__( tags=child_tags, ) + # Add diagnostic settings for firewall + # This links the firewall to the log analytics workspace + insights.DiagnosticSettings( + f"{self._name}_firewall_diagnostic_settings", + name="firewall_diagnostic_settings", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "AllMetrics", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=firewall.id, + workspace_id=props.log_analytics_workspace.workspace_id, + ) + # Retrieve the private IP address for the firewall private_ip_address = firewall.ip_configurations.apply( lambda cfgs: "" if not cfgs else cfgs[0].private_ip_address From a09802a36b77c29f8198dc8ca6a5dd08cea0ca77 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 25 Nov 2024 16:33:29 +0000 Subject: [PATCH 047/100] Correct component name --- data_safe_haven/infrastructure/programs/sre/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 1f46db980b..4e45aff208 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -336,7 +336,7 @@ def __init__( # Add diagnostic settings for firewall # This links the firewall to the log analytics workspace - insights.DiagnosticSettings( + insights.DiagnosticSetting( f"{self._name}_firewall_diagnostic_settings", name="firewall_diagnostic_settings", log_analytics_destination_type="Dedicated", From d886b102c5c01f3b0dc8e5ce8f575cf50b19b2be Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:10:07 +0000 Subject: [PATCH 048/100] Clarify error message and check against SRE context --- data_safe_haven/commands/users.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index d479a1393e..01dd79e6c8 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -9,8 +9,8 @@ from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi -from data_safe_haven.logging import get_logger from data_safe_haven.infrastructure import SREProjectManager +from data_safe_haven.logging import get_logger users_command_group = typer.Typer() @@ -122,9 +122,8 @@ def register( try: shm_config = SHMConfig.from_remote(context) except DataSafeHavenError: - msg = "Have you deployed the SHM?" - logger.error(msg) - raise DataSafeHavenError(msg) + logger.error("Have you deployed the SHM?") + raise typer.Exit(1) # Load Pulumi config pulumi_config = DSHPulumiConfig.from_remote(context) @@ -154,7 +153,6 @@ def register( # List users users = UserHandler(context, graph_api) - # available_usernames = users.get_usernames_entra_id() available_users = users.entra_users.list() user_dict = { user.user_principal_name.split("@")[0]: user.user_principal_name.split("@")[ @@ -163,16 +161,15 @@ def register( for user in available_users } usernames_to_register = [] - shm_name = sre_stack.output("linked_shm")["name"] + shm_name = sre_stack.output("context") for username in usernames: if username in user_dict.keys(): - user_domain = user_dict[username] + user_domain = user_dict[username].split(".")[0] if shm_name not in user_domain: logger.error( - f"Username '{username}' belongs to SHM domain '{user_domain}'.\n" - f"SRE '{sre_config.name}' is associated with SHM domain '{shm_name}'.\n" - "Users can only be registered to one SHM domain.\n" - "Please use 'dsh users add' to create a new user associated with the current SHM domain." + f"Username [green]'{username}'[/green] belongs to SHM context [blue]'{user_domain}'[/blue].\n" + f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM context [blue]'{shm_name}'[/blue].\n" + "The user must belong to the same SHM context as the SRE." ) else: usernames_to_register.append(username) From 01585a1b0c1a283fd594803873be81dead49913b Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:10:54 +0000 Subject: [PATCH 049/100] Fix linting --- data_safe_haven/commands/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 01dd79e6c8..b3456c8784 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -121,9 +121,9 @@ def register( # Load SHMConfig try: shm_config = SHMConfig.from_remote(context) - except DataSafeHavenError: + except DataSafeHavenError as exc: logger.error("Have you deployed the SHM?") - raise typer.Exit(1) + raise typer.Exit(1) from exc # Load Pulumi config pulumi_config = DSHPulumiConfig.from_remote(context) From d5042b2cd262630f4c6a644c1707fd238e178436 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:11:17 +0000 Subject: [PATCH 050/100] Export context name associated with SRE as part of SRE stack --- data_safe_haven/infrastructure/programs/declarative_sre.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 6ce9c81d1d..d9d6f0b545 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -424,7 +424,7 @@ def __call__(self) -> None: # Export values for later use pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) + pulumi.export("context", self.context.name) pulumi.export("remote_desktop", remote_desktop.exports) pulumi.export("sre_fqdn", networking.sre_fqdn) - pulumi.export("linked_shm", self.context.name) pulumi.export("workspaces", workspaces.exports) From 134404b068255d281d6567c349267361b15d6104 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:41:00 +0000 Subject: [PATCH 051/100] Correct list indent --- .github/ISSUE_TEMPLATE/release_checklist.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index ade631052e..8686c95238 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -34,10 +34,10 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo ### For major releases only - [ ] Confirm that a third party has carried out a full penetration test evaluating: - 1. external attack surface - 1. ability to exfiltrate data from the system - 1. ability to transfer data between SREs - 1. ability to escalate privileges on the SRD. + 1. external attack surface + 1. ability to exfiltrate data from the system + 1. ability to transfer data between SREs + 1. ability to escalate privileges on the SRD. ### Update documentation From bb0eaf2c391781bb0219ad76b427a235162f98f8 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:48:49 +0000 Subject: [PATCH 052/100] Remove bare URL --- .github/ISSUE_TEMPLATE/release_checklist.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 8686c95238..42acf2fe51 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -14,9 +14,9 @@ Before reporting a problem please check the following. Replace the empty checkbo Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deployment) section of our documentation when completing these steps. -- [ ] Consult the `data-safe-haven/VERSIONING.md` guide and determine the version number of the new release. Record it in the title of this issue. +- [ ] Consult the `data-safe-haven/VERSIONING.md` guide and determine the version number of the new release. Record it in the title of this issue - [ ] Create a release branch called e.g. `release-v0.0.1` -- [ ] Draft a changelog for the release similar to our previous releases, see https://github.com/alan-turing-institute/data-safe-haven/releases +- [ ] Draft a changelog for the release similar to our [previous releases](https://github.com/alan-turing-institute/data-safe-haven/releases) ### For patch releases only From 56081d87bff505215239d3f19f8f46a0483ea28c Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:49:04 +0000 Subject: [PATCH 053/100] Clarify tag and release creation --- .github/ISSUE_TEMPLATE/release_checklist.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 42acf2fe51..575f5c9c53 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -47,7 +47,8 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo ### Making the release - [ ] Merge release branch into `latest` -- [ ] Push tag in the format `v0.0.1` to the merge commit into `latest` +- [ ] Create a tag of the form `v0.0.1` pointing to the most recent commit on `latest` (the merge that you just made) +- [ ] Publish your draft GitHub release using this tag - [ ] Ensure docs for the latest version are built and deployed on ReadTheDocs - [ ] Push a build to PyPI - [ ] Announce release on communications channels From d020b3e7ed764a42167e80adba37b1d854119a70 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 09:58:48 +0000 Subject: [PATCH 054/100] Use full resource URI --- data_safe_haven/infrastructure/programs/sre/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/firewall.py b/data_safe_haven/infrastructure/programs/sre/firewall.py index 4e45aff208..ed831e826a 100644 --- a/data_safe_haven/infrastructure/programs/sre/firewall.py +++ b/data_safe_haven/infrastructure/programs/sre/firewall.py @@ -361,7 +361,7 @@ def __init__( } ], resource_uri=firewall.id, - workspace_id=props.log_analytics_workspace.workspace_id, + workspace_id=props.log_analytics_workspace.id, ) # Retrieve the private IP address for the firewall From 757dcf1a3a7f0ea292aa9adb8e694cc8246287ff Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:11:03 +0000 Subject: [PATCH 055/100] Use SHM FQDN in error message rather than context name, clarify error message --- data_safe_haven/commands/users.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index b3456c8784..2db3cf5bd3 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -164,19 +164,19 @@ def register( shm_name = sre_stack.output("context") for username in usernames: if username in user_dict.keys(): - user_domain = user_dict[username].split(".")[0] + user_domain = user_dict[username] if shm_name not in user_domain: logger.error( - f"Username [green]'{username}'[/green] belongs to SHM context [blue]'{user_domain}'[/blue].\n" - f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM context [blue]'{shm_name}'[/blue].\n" - "The user must belong to the same SHM context as the SRE." + f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" + f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" + "The user's principal domain name must match the domain of the SRE to be registered." ) else: usernames_to_register.append(username) else: logger.error( f"Username '{username}' does not belong to this Data Safe Haven deployment." - " Please use 'dsh users add' to create it." + "Please use 'dsh users add' to create this user." ) users.register(sre_config.name, usernames_to_register) except DataSafeHavenError as exc: From 510a7149983c2cb97f9bd0d5a7d3d4b04a4c7fdb Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:15:46 +0000 Subject: [PATCH 056/100] Remove unneeded export of context name --- data_safe_haven/commands/users.py | 3 +-- data_safe_haven/infrastructure/programs/declarative_sre.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 2db3cf5bd3..e9e96a5283 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -161,11 +161,10 @@ def register( for user in available_users } usernames_to_register = [] - shm_name = sre_stack.output("context") for username in usernames: if username in user_dict.keys(): user_domain = user_dict[username] - if shm_name not in user_domain: + if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index d9d6f0b545..78467f201b 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -424,7 +424,6 @@ def __call__(self) -> None: # Export values for later use pulumi.export("data", data.exports) pulumi.export("ldap", ldap_group_names) - pulumi.export("context", self.context.name) pulumi.export("remote_desktop", remote_desktop.exports) pulumi.export("sre_fqdn", networking.sre_fqdn) pulumi.export("workspaces", workspaces.exports) From 309d777298356b8f1070909dd4409c76ee3233b8 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 10:29:38 +0000 Subject: [PATCH 057/100] Remove unnecessary loading of SRE stack --- data_safe_haven/commands/users.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index e9e96a5283..5dab2e7591 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -9,7 +9,6 @@ from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError from data_safe_haven.external import GraphApi -from data_safe_haven.infrastructure import SREProjectManager from data_safe_haven.logging import get_logger users_command_group = typer.Typer() @@ -135,12 +134,6 @@ def register( logger.error(msg) raise typer.Exit(1) - sre_stack = SREProjectManager( - context=context, - config=sre_config, - pulumi_config=pulumi_config, - ) - # Load GraphAPI graph_api = GraphApi.from_scopes( scopes=["Group.ReadWrite.All", "GroupMember.ReadWrite.All"], From fa08bc3a5d324d231b7e7b759ed53117a220410f Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 10:58:31 +0000 Subject: [PATCH 058/100] Add documentation for firewall logs --- docs/source/management/logs.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index f9a9948453..5d52fe5d77 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -21,7 +21,7 @@ These include, - Gitea and Hedgedoc Logs from all containers are ingested into the [SREs log workspace](#log-workspace). -There are two logs +There are two tables, `ContainerEvents_CL` : Event logs for the container instance resources such as starting, stopping, crashes and pulling images. @@ -29,3 +29,22 @@ There are two logs `ContainerInstanceLog_CL` : Container process logs. : This is where you can view the output of the containerised applications and will be useful for debugging problems. + +## Firewall logs + +The firewall plays a critical role in the security of a Data Safe Haven. +It filters all outbound traffic through a set of FQDN rules so that each component may only reach necessary and allowed domains. + +Logs from the firewall are ingested into the [SREs log workspace](#log-workspace). +There are multiple tables, + +`AZFWApplicationRule` +: Logs from the firewalls FDQN filters. +: Shows requests to the outside of the Data Safe Haven and why they have been approved or rejected. + +`AZFWDnsQuery` +: DNS requests handled by the firewall. + +`AzureMetrics` +: Various metrics on firewall utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. From fd6dfec1f5330fb2b014dd79d92cf09f66e3ace0 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 11:57:32 +0000 Subject: [PATCH 059/100] Add diagnostic setting for NFSv3 containers --- .../composite/nfsv3_blob_container.py | 44 ++++++++++++++++++- .../infrastructure/programs/sre/data.py | 5 +++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index 98564918a0..f4275fed8e 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,9 +1,10 @@ from pulumi import ComponentResource, Input, ResourceOptions -from pulumi_azure_native import storage +from pulumi_azure_native import insights, storage -from data_safe_haven.infrastructure.components.dynamic.blob_container_acl import ( +from data_safe_haven.infrastructure.components import ( BlobContainerAcl, BlobContainerAclProps, + WrappedLogAnalyticsWorkspace, ) @@ -15,6 +16,7 @@ def __init__( acl_other: Input[str], apply_default_permissions: Input[bool], container_name: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account: Input[storage.StorageAccount], subscription_name: Input[str], @@ -24,6 +26,7 @@ def __init__( self.acl_other = acl_other self.apply_default_permissions = apply_default_permissions self.container_name = container_name + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account = storage_account self.subscription_name = subscription_name @@ -52,6 +55,7 @@ def __init__( ResourceOptions(parent=props.storage_account), ), ) + BlobContainerAcl( f"{storage_container._name}_acl", BlobContainerAclProps( @@ -70,6 +74,42 @@ def __init__( ), ) + insights.DiagnosticSetting( + f"{storage_container._name}_diagnostic_settings", + name="firewall_diagnostic_settings", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_container.id, + workspace_id=props.log_analytics_workspace.id, + ) + self.name = storage_container.name self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 711b76139f..f478b56df8 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -33,6 +33,7 @@ NFSV3BlobContainerProps, SSLCertificate, SSLCertificateProps, + WrappedLogAnalyticsWorkspace, WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -51,6 +52,7 @@ def __init__( dns_record: Input[network.RecordSet], dns_server_admin_password: Input[pulumi_random.RandomPassword], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], storage_quota_gb_home: Input[int], @@ -69,6 +71,7 @@ def __init__( self.dns_record = dns_record self.password_dns_server_admin = dns_server_admin_password self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -492,6 +495,7 @@ def __init__( # 65533 ownership of the fileshare (preventing use inside the SRE) apply_default_permissions=False, container_name="egress", + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, @@ -507,6 +511,7 @@ def __init__( # files (eg. with Azure Storage Explorer) apply_default_permissions=True, container_name="ingress", + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, From bd045eaf8c33fdcf86905906cb3a31a2f49a5328 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 12:52:18 +0000 Subject: [PATCH 060/100] user preferred_name instead of directly accessing principal name --- data_safe_haven/commands/users.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 5dab2e7591..53473f58ed 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -148,9 +148,7 @@ def register( users = UserHandler(context, graph_api) available_users = users.entra_users.list() user_dict = { - user.user_principal_name.split("@")[0]: user.user_principal_name.split("@")[ - 1 - ] + user.preferred_username.split("@")[0]: user.preferred_username.split("@")[1] for user in available_users } usernames_to_register = [] From 6dc5a688bf0efcf5f1ec1c053c14f26d78b3441e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 14:27:24 +0000 Subject: [PATCH 061/100] Correct imports --- .../components/composite/nfsv3_blob_container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index f4275fed8e..96d6e8a8b0 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,9 +1,11 @@ from pulumi import ComponentResource, Input, ResourceOptions from pulumi_azure_native import insights, storage -from data_safe_haven.infrastructure.components import ( +from data_safe_haven.infrastructure.components.dynamic import ( BlobContainerAcl, BlobContainerAclProps, +) +from data_safe_haven.infrastructure.components.wrapped import ( WrappedLogAnalyticsWorkspace, ) From 9bc497f23b05d9a7ebd665e8f3d73280fde07ed5 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 14:36:10 +0000 Subject: [PATCH 062/100] Pass log workspace to data component --- .../programs/declarative_sre.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 78467f201b..96df9380b2 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -183,6 +183,20 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy data storage data = SREDataComponent( "sre_data", @@ -196,6 +210,7 @@ def __call__(self) -> None: dns_record=networking.shm_ns_record, dns_server_admin_password=dns.password_admin, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, storage_quota_gb_home=self.config.sre.storage_quota_gb.home, @@ -209,20 +224,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy the apt proxy server apt_proxy_server = SREAptProxyServerComponent( "sre_apt_proxy_server", From 78754ae57421113b1c8f95d85926c64380e3dd69 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 15:35:53 +0000 Subject: [PATCH 063/100] Move diagnostic setting to storage account --- .../composite/nfsv3_blob_container.py | 43 +------------------ .../infrastructure/programs/sre/data.py | 41 +++++++++++++++++- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index 96d6e8a8b0..29550e9541 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,13 +1,10 @@ from pulumi import ComponentResource, Input, ResourceOptions -from pulumi_azure_native import insights, storage +from pulumi_azure_native import storage from data_safe_haven.infrastructure.components.dynamic import ( BlobContainerAcl, BlobContainerAclProps, ) -from data_safe_haven.infrastructure.components.wrapped import ( - WrappedLogAnalyticsWorkspace, -) class NFSV3BlobContainerProps: @@ -18,7 +15,6 @@ def __init__( acl_other: Input[str], apply_default_permissions: Input[bool], container_name: Input[str], - log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account: Input[storage.StorageAccount], subscription_name: Input[str], @@ -28,7 +24,6 @@ def __init__( self.acl_other = acl_other self.apply_default_permissions = apply_default_permissions self.container_name = container_name - self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account = storage_account self.subscription_name = subscription_name @@ -76,42 +71,6 @@ def __init__( ), ) - insights.DiagnosticSetting( - f"{storage_container._name}_diagnostic_settings", - name="firewall_diagnostic_settings", - log_analytics_destination_type="Dedicated", - logs=[ - { - "category_group": "allLogs", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - { - "category_group": "audit", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - ], - metrics=[ - { - "category": "Transaction", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - } - ], - resource_uri=storage_container.id, - workspace_id=props.log_analytics_workspace.id, - ) - self.name = storage_container.name self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index f478b56df8..fcca697810 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,6 +7,7 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, + insights, keyvault, managedidentity, network, @@ -495,7 +496,6 @@ def __init__( # 65533 ownership of the fileshare (preventing use inside the SRE) apply_default_permissions=False, container_name="egress", - log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, @@ -511,12 +511,49 @@ def __init__( # files (eg. with Azure Storage Explorer) apply_default_permissions=True, container_name="ingress", - log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, ), ) + # Add diagnostic setting for blobs + insights.DiagnosticSetting( + f"{storage_account_data_private_sensitive._name}_diagnostic_setting", + name=f"{storage_account_data_private_sensitive._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_private_sensitive.id.apply( + lambda resource_id: resource_id + "/blobServices" + ), + workspace_id=props.log_analytics_workspace.id, + ) # Set up a private endpoint for the sensitive data storage account storage_account_data_private_sensitive_endpoint = network.PrivateEndpoint( f"{storage_account_data_private_sensitive._name}_private_endpoint", From 469f1d6ac57bb81e8e4bebb7b4a84d897c7af163 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:40:04 +0000 Subject: [PATCH 064/100] better formatted error message --- data_safe_haven/commands/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 53473f58ed..b3a90c0664 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -158,14 +158,14 @@ def register( if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" - f"SRE [yellow]'{sre_config.name}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" + f"SRE [yellow]'{sre}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" "The user's principal domain name must match the domain of the SRE to be registered." ) else: usernames_to_register.append(username) else: logger.error( - f"Username '{username}' does not belong to this Data Safe Haven deployment." + f"Username '{username}' does not belong to this Data Safe Haven deployment.\n" "Please use 'dsh users add' to create this user." ) users.register(sre_config.name, usernames_to_register) From 2abd14b27984ac2df81b3da60ffdf8d537927019 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 15:40:52 +0000 Subject: [PATCH 065/100] add test for mismatched domain --- tests/commands/conftest.py | 13 +++++++++++++ tests/commands/test_users.py | 20 ++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tests/commands/conftest.py b/tests/commands/conftest.py index d675398bfc..de60eb29d0 100644 --- a/tests/commands/conftest.py +++ b/tests/commands/conftest.py @@ -1,6 +1,8 @@ from pytest import fixture from typer.testing import CliRunner +from data_safe_haven.administration.users.entra_users import EntraUsers +from data_safe_haven.administration.users.research_user import ResearchUser from data_safe_haven.config import ( Context, ContextManager, @@ -260,3 +262,14 @@ def tmp_contexts_none(tmp_path, context_yaml): with open(config_file_path, "w") as f: f.write(context_yaml) return tmp_path + + +@fixture +def mock_entra_user_list(mocker): + test_user = ResearchUser( + given_name="Harry", + surname="Lime", + sam_account_name="harry.lime", + user_principal_name="harry.lime@acme.testing", + ) + mocker.patch.object(EntraUsers, "list", return_value=[test_user]) diff --git a/tests/commands/test_users.py b/tests/commands/test_users.py index c1b183c922..5c11e29cc9 100644 --- a/tests/commands/test_users.py +++ b/tests/commands/test_users.py @@ -52,6 +52,26 @@ def test_invalid_shm( assert result.exit_code == 1 assert "Have you deployed the SHM?" in result.stdout + def test_mismatched_domain( + self, + mock_graphapi_get_credential, # noqa: ARG002 + mock_pulumi_config_no_key_from_remote, # noqa: ARG002 + mock_shm_config_from_remote, # noqa: ARG002 + mock_sre_config_from_remote, # noqa: ARG002 + mock_entra_user_list, # noqa: ARG002 + runner, + tmp_contexts, # noqa: ARG002 + ): + result = runner.invoke( + users_command_group, ["register", "-u", "harry.lime", "sandbox"] + ) + + assert result.exit_code == 0 + assert ( + "principal domain name must match the domain of the SRE to be registered" + in result.stdout + ) + def test_invalid_sre( self, mock_pulumi_config_from_remote, # noqa: ARG002 From 23926e5d3c4151c50d4b218844db493cd41f3e9d Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:12:11 +0000 Subject: [PATCH 066/100] Add workspace log docs --- docs/source/management/logs.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index f9a9948453..0b575fde2d 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -20,7 +20,7 @@ These include, - package proxy - Gitea and Hedgedoc -Logs from all containers are ingested into the [SREs log workspace](#log-workspace). +Logs from all containers are ingested into the [SRE's log analytics workspace](#log-workspace). There are two logs `ContainerEvents_CL` @@ -29,3 +29,18 @@ There are two logs `ContainerInstanceLog_CL` : Container process logs. : This is where you can view the output of the containerised applications and will be useful for debugging problems. + +## Workspace logs + +Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace). + +There are three logs + +`Perf` +: Usage statistics for individual workspaces, such as percent memory used and percent disk space used + +`Syslog` +: Linux system logs for individual workspaces, useful for debugging problems related to system processes + +`Heartbeat` +: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) \ No newline at end of file From 52f587ac824cee29652d63b8686f066272c7e8ca Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:13:12 +0000 Subject: [PATCH 067/100] fix linting --- docs/source/management/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 0b575fde2d..3c5b9c7b18 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -43,4 +43,4 @@ There are three logs : Linux system logs for individual workspaces, useful for debugging problems related to system processes `Heartbeat` -: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) \ No newline at end of file +: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) From dbcd646ce830ab31da58144d63bc682d03427c99 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:14:40 +0000 Subject: [PATCH 068/100] Add link to docs for Azure Monitor Agent --- docs/source/management/logs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 3c5b9c7b18..046f9c12eb 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -7,7 +7,7 @@ Some of these logs are ingested into a central location, an Azure [Log Analytics Each SRE has its own Log Analytics Workspace. You can view the workspaces by going to the Azure portal and navigating to [Log Analytics Workspaces](https://portal.azure.com/#browse/Microsoft.OperationalInsights%2Fworkspaces). -Select which log workspace you want to view by clicking on the workspace named `shm--sre--log`. +Select which Log Analytics Workspace you want to view by clicking on the workspace named `shm--sre--log`. The logs can be filtered using [Kusto Query Language (KQL)](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview). @@ -32,7 +32,7 @@ There are two logs ## Workspace logs -Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace). +Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace) using the [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/azure-monitor-agent-overview). There are three logs @@ -43,4 +43,4 @@ There are three logs : Linux system logs for individual workspaces, useful for debugging problems related to system processes `Heartbeat` -: Verification that the Azure Monitoring Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) +: Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) From 6c9fb1759630c300c764fbee78762154e0133481 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 09:47:20 +0000 Subject: [PATCH 069/100] Correct T2/3 PyPI/CRAN proxy information --- docs/source/overview/sensitivity_tiers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/overview/sensitivity_tiers.md b/docs/source/overview/sensitivity_tiers.md index 4aef9a32fe..995be6ab87 100644 --- a/docs/source/overview/sensitivity_tiers.md +++ b/docs/source/overview/sensitivity_tiers.md @@ -49,7 +49,7 @@ Non-technical restrictions related to information governance procedures may also - connections to the in-browser remote desktop can only be made from an agreed set of IP addresses - outbound connections to the internet from inside the environment are not possible - copy-and-paste between the environment and the user's device is not possible -- access to all packages on PyPI and CRAN is made available through a proxy or mirror server +- access to all packages on PyPI and CRAN is made available through a proxy server Non-technical restrictions related to information governance procedures may also be applied according to your organisation's needs. @@ -63,7 +63,7 @@ At the Turing connections to Tier 2 environments are only permitted from **Organ **Tier 3** environments impose the following technical controls on top of what is required at {ref}`policy_tier_2`. -- a partial replica of agreed PyPI and CRAN packages is made available through a proxy or mirror server +- an agreed subset of PyPI and CRAN packages is made available through a proxy server Non-technical restrictions related to information governance procedures may also be applied according to your organisation's needs. From 8222d646a49933dbad233cf6abe22c9a98f6a6f6 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 09:52:18 +0000 Subject: [PATCH 070/100] Add full stops --- docs/source/management/logs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 046f9c12eb..4fca60864b 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -37,10 +37,10 @@ Logs from all user workspaces are ingested into the [SRE's log analytics workspa There are three logs `Perf` -: Usage statistics for individual workspaces, such as percent memory used and percent disk space used +: Usage statistics for individual workspaces, such as percent memory used and percent disk space used. `Syslog` -: Linux system logs for individual workspaces, useful for debugging problems related to system processes +: Linux system logs for individual workspaces, useful for debugging problems related to system processes. `Heartbeat` -: Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace) +: Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace). From 7088eea3f1f49848f55a71b1e79beb7c7caaac2e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 10:09:41 +0000 Subject: [PATCH 071/100] Adjust syslog description --- docs/source/management/logs.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 4fca60864b..cf55508cd3 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -40,7 +40,9 @@ There are three logs : Usage statistics for individual workspaces, such as percent memory used and percent disk space used. `Syslog` -: Linux system logs for individual workspaces, useful for debugging problems related to system processes. +: [syslog](https://www.paessler.com/it-explained/syslog) events from workspaces. +: Syslog is the _de facto_ standard protocol for logging on Linux and most applications will log to it. +: These logs will be useful for debugging problems with the workspace or workspace software. `Heartbeat` : Verification that the Azure Monitor Agent is present on the workspaces and is able to connect to the [log analytics workspace](#log-workspace). From 168ca245117aa1ff906a1319aab4ad9c11339ccd Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 11:01:54 +0000 Subject: [PATCH 072/100] Correct blobServices URI --- data_safe_haven/infrastructure/programs/sre/data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index fcca697810..5f68a8f637 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -550,7 +550,9 @@ def __init__( } ], resource_uri=storage_account_data_private_sensitive.id.apply( - lambda resource_id: resource_id + "/blobServices" + # This is the URI of the blobServices resource which is automatically + # created. + lambda resource_id: resource_id + "/blobServices/default" ), workspace_id=props.log_analytics_workspace.id, ) From 7887e0987ea575a8c2940cdb1014a7de0bf19a68 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 11:43:12 +0000 Subject: [PATCH 073/100] Move NFSv3 accounts to a component resource Diagnostic settings are created as part of the component. --- .../infrastructure/components/__init__.py | 6 +- .../components/composite/__init__.py | 6 + .../composite/nfsv3_storage_account.py | 144 ++++++++++++++++++ .../components/wrapped/__init__.py | 2 - .../wrapped/nfsv3_storage_account.py | 79 ---------- .../programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/data.py | 70 +++------ .../programs/sre/desired_state.py | 28 ++-- 8 files changed, 191 insertions(+), 145 deletions(-) create mode 100644 data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py delete mode 100644 data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py diff --git a/data_safe_haven/infrastructure/components/__init__.py b/data_safe_haven/infrastructure/components/__init__.py index f4b93b9c3d..52043d1ad3 100644 --- a/data_safe_haven/infrastructure/components/__init__.py +++ b/data_safe_haven/infrastructure/components/__init__.py @@ -9,6 +9,8 @@ MicrosoftSQLDatabaseProps, NFSV3BlobContainerComponent, NFSV3BlobContainerProps, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, PostgresqlDatabaseComponent, PostgresqlDatabaseProps, VMComponent, @@ -23,7 +25,6 @@ ) from .wrapped import ( WrappedLogAnalyticsWorkspace, - WrappedNFSV3StorageAccount, ) __all__ = [ @@ -41,11 +42,12 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", + "NFSV3StorageAccountComponent", + "NFSV3StorageAccountProps", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "SSLCertificate", "SSLCertificateProps", "VMComponent", "WrappedLogAnalyticsWorkspace", - "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/composite/__init__.py b/data_safe_haven/infrastructure/components/composite/__init__.py index bc09bc18a8..8e561dd73a 100644 --- a/data_safe_haven/infrastructure/components/composite/__init__.py +++ b/data_safe_haven/infrastructure/components/composite/__init__.py @@ -9,6 +9,10 @@ MicrosoftSQLDatabaseProps, ) from .nfsv3_blob_container import NFSV3BlobContainerComponent, NFSV3BlobContainerProps +from .nfsv3_storage_account import ( + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, +) from .postgresql_database import PostgresqlDatabaseComponent, PostgresqlDatabaseProps from .virtual_machine import LinuxVMComponentProps, VMComponent @@ -23,6 +27,8 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", + "NFSV3StorageAccountComponent", + "NFSV3StorageAccountProps", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "VMComponent", diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py b/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py new file mode 100644 index 0000000000..ca003bbd3d --- /dev/null +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py @@ -0,0 +1,144 @@ +from collections.abc import Mapping, Sequence + +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import insights, storage + +from data_safe_haven.external import AzureIPv4Range +from data_safe_haven.infrastructure.components.wrapped import ( + WrappedLogAnalyticsWorkspace, +) +from data_safe_haven.types import AzureServiceTag + + +class NFSV3StorageAccountProps: + def __init__( + self, + account_name: Input[str], + allowed_ip_addresses: Input[Sequence[str]] | None, + allowed_service_tag: AzureServiceTag | None, + location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], + resource_group_name: Input[str], + subnet_id: Input[str], + ): + self.account_name = account_name + self.allowed_ip_addresses = allowed_ip_addresses + self.allowed_service_tag = allowed_service_tag + self.location = location + self.log_analytics_workspace = log_analytics_workspace + self.resource_group_name = resource_group_name + self.subnet_id = subnet_id + + +class NFSV3StorageAccountComponent(ComponentResource): + encryption_args = storage.EncryptionArgs( + key_source=storage.KeySource.MICROSOFT_STORAGE, + services=storage.EncryptionServicesArgs( + blob=storage.EncryptionServiceArgs( + enabled=True, key_type=storage.KeyType.ACCOUNT + ), + file=storage.EncryptionServiceArgs( + enabled=True, key_type=storage.KeyType.ACCOUNT + ), + ), + ) + + def __init__( + self, + name: str, + props: NFSV3StorageAccountProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ): + super().__init__("dsh:sre:NFSV3StorageAccountComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = {"component": "data"} | (tags if tags else {}) + + if props.allowed_service_tag == AzureServiceTag.INTERNET: + default_action = storage.DefaultAction.ALLOW + ip_rules = [] + else: + default_action = storage.DefaultAction.DENY + ip_rules = Output.from_input(props.allowed_ip_addresses).apply( + lambda ip_ranges: [ + storage.IPRuleArgs( + action=storage.Action.ALLOW, + i_p_address_or_range=str(ip_address), + ) + for ip_range in sorted(ip_ranges) + for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips() + ] + ) + + # Deploy storage account + self.storage_account = storage.StorageAccount( + f"{self._name}", + account_name=props.account_name, + allow_blob_public_access=False, + enable_https_traffic_only=True, + enable_nfs_v3=True, + encryption=self.encryption_args, + is_hns_enabled=True, + kind=storage.Kind.BLOCK_BLOB_STORAGE, + location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, + network_rule_set=storage.NetworkRuleSetArgs( + bypass=storage.Bypass.AZURE_SERVICES, + default_action=default_action, + ip_rules=ip_rules, + virtual_network_rules=[ + storage.VirtualNetworkRuleArgs( + virtual_network_resource_id=props.subnet_id, + ) + ], + ), + public_network_access=storage.PublicNetworkAccess.ENABLED, + resource_group_name=props.resource_group_name, + sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS), + opts=child_opts, + tags=child_tags, + ) + + # Add diagnostic setting for blobs + insights.DiagnosticSetting( + f"{self.storage_account._name}_diagnostic_setting", + name=f"{self.storage_account._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=self.storage_account.id.apply( + # This is the URI of the blobServices resource which is automatically + # created. + lambda resource_id: resource_id + + "/blobServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) + + self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/components/wrapped/__init__.py b/data_safe_haven/infrastructure/components/wrapped/__init__.py index b449f46859..fc5f8c8f61 100644 --- a/data_safe_haven/infrastructure/components/wrapped/__init__.py +++ b/data_safe_haven/infrastructure/components/wrapped/__init__.py @@ -1,7 +1,5 @@ from .log_analytics_workspace import WrappedLogAnalyticsWorkspace -from .nfsv3_storage_account import WrappedNFSV3StorageAccount __all__ = [ "WrappedLogAnalyticsWorkspace", - "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py b/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py deleted file mode 100644 index e259de4806..0000000000 --- a/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py +++ /dev/null @@ -1,79 +0,0 @@ -from collections.abc import Mapping, Sequence - -from pulumi import Input, Output, ResourceOptions -from pulumi_azure_native import storage - -from data_safe_haven.external import AzureIPv4Range -from data_safe_haven.types import AzureServiceTag - - -class WrappedNFSV3StorageAccount(storage.StorageAccount): - encryption_args = storage.EncryptionArgs( - key_source=storage.KeySource.MICROSOFT_STORAGE, - services=storage.EncryptionServicesArgs( - blob=storage.EncryptionServiceArgs( - enabled=True, key_type=storage.KeyType.ACCOUNT - ), - file=storage.EncryptionServiceArgs( - enabled=True, key_type=storage.KeyType.ACCOUNT - ), - ), - ) - - def __init__( - self, - resource_name: str, - *, - account_name: Input[str], - allowed_ip_addresses: Input[Sequence[str]] | None, - allowed_service_tag: AzureServiceTag | None, - location: Input[str], - resource_group_name: Input[str], - subnet_id: Input[str], - opts: ResourceOptions, - tags: Input[Mapping[str, Input[str]]], - ): - if allowed_service_tag == AzureServiceTag.INTERNET: - default_action = storage.DefaultAction.ALLOW - ip_rules = [] - else: - default_action = storage.DefaultAction.DENY - ip_rules = Output.from_input(allowed_ip_addresses).apply( - lambda ip_ranges: [ - storage.IPRuleArgs( - action=storage.Action.ALLOW, - i_p_address_or_range=str(ip_address), - ) - for ip_range in sorted(ip_ranges) - for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips() - ] - ) - - self.resource_group_name_ = Output.from_input(resource_group_name) - super().__init__( - resource_name, - account_name=account_name, - allow_blob_public_access=False, - enable_https_traffic_only=True, - enable_nfs_v3=True, - encryption=self.encryption_args, - is_hns_enabled=True, - kind=storage.Kind.BLOCK_BLOB_STORAGE, - location=location, - minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, - network_rule_set=storage.NetworkRuleSetArgs( - bypass=storage.Bypass.AZURE_SERVICES, - default_action=default_action, - ip_rules=ip_rules, - virtual_network_rules=[ - storage.VirtualNetworkRuleArgs( - virtual_network_resource_id=subnet_id, - ) - ], - ), - public_network_access=storage.PublicNetworkAccess.ENABLED, - resource_group_name=resource_group_name, - sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS), - opts=opts, - tags=tags, - ) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 96df9380b2..da796adfcc 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -377,6 +377,7 @@ def __call__(self) -> None: ldap_user_filter=ldap_user_filter, ldap_user_search_base=ldap_user_search_base, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, software_repository_hostname=user_services.software_repositories.hostname, subnet_desired_state=networking.subnet_desired_state, diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 5f68a8f637..645ea8fd87 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,7 +7,6 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, - insights, keyvault, managedidentity, network, @@ -32,10 +31,11 @@ from data_safe_haven.infrastructure.components import ( NFSV3BlobContainerComponent, NFSV3BlobContainerProps, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, SSLCertificate, SSLCertificateProps, WrappedLogAnalyticsWorkspace, - WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -471,20 +471,26 @@ def __init__( # Deploy sensitive data blob storage account # - This holds the /mnt/input and /mnt/output containers that are mounted by workspaces # - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer - storage_account_data_private_sensitive = WrappedNFSV3StorageAccount( + component_data_private_sensitive = NFSV3StorageAccountComponent( f"{self._name}_storage_account_data_private_sensitive", - # Storage account names have a maximum of 24 characters - account_name=alphanumeric( - f"{''.join(truncate_tokens(stack_name.split('-'), 11))}sensitivedata{sha256hash(self._name)}" - )[:24], - allowed_ip_addresses=data_private_sensitive_ip_addresses, - allowed_service_tag=data_private_sensitive_service_tag, - location=props.location, - subnet_id=props.subnet_data_private_id, - resource_group_name=props.resource_group_name, + NFSV3StorageAccountProps( + # Storage account names have a maximum of 24 characters + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 11))}sensitivedata{sha256hash(self._name)}" + )[:24], + allowed_ip_addresses=data_private_sensitive_ip_addresses, + allowed_service_tag=data_private_sensitive_service_tag, + location=props.location, + log_analytics_workspace=props.log_analytics_workspace, + subnet_id=props.subnet_data_private_id, + resource_group_name=props.resource_group_name, + ), opts=child_opts, tags=child_tags, ) + storage_account_data_private_sensitive = ( + component_data_private_sensitive.storage_account + ) # Deploy storage containers NFSV3BlobContainerComponent( f"{self._name}_blob_egress", @@ -516,46 +522,6 @@ def __init__( subscription_name=props.subscription_name, ), ) - # Add diagnostic setting for blobs - insights.DiagnosticSetting( - f"{storage_account_data_private_sensitive._name}_diagnostic_setting", - name=f"{storage_account_data_private_sensitive._name}_diagnostic_setting", - log_analytics_destination_type="Dedicated", - logs=[ - { - "category_group": "allLogs", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - { - "category_group": "audit", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - ], - metrics=[ - { - "category": "Transaction", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - } - ], - resource_uri=storage_account_data_private_sensitive.id.apply( - # This is the URI of the blobServices resource which is automatically - # created. - lambda resource_id: resource_id + "/blobServices/default" - ), - workspace_id=props.log_analytics_workspace.id, - ) # Set up a private endpoint for the sensitive data storage account storage_account_data_private_sensitive_endpoint = network.PrivateEndpoint( f"{storage_account_data_private_sensitive._name}_private_endpoint", diff --git a/data_safe_haven/infrastructure/programs/sre/desired_state.py b/data_safe_haven/infrastructure/programs/sre/desired_state.py index c4392f5210..20f4e357f1 100644 --- a/data_safe_haven/infrastructure/programs/sre/desired_state.py +++ b/data_safe_haven/infrastructure/programs/sre/desired_state.py @@ -31,7 +31,9 @@ from data_safe_haven.infrastructure.components import ( NFSV3BlobContainerComponent, NFSV3BlobContainerProps, - WrappedNFSV3StorageAccount, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.types import AzureDnsZoneNames @@ -55,6 +57,7 @@ def __init__( ldap_user_filter: Input[str], ldap_user_search_base: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], software_repository_hostname: Input[str], subscription_name: Input[str], @@ -73,6 +76,7 @@ def __init__( self.ldap_user_filter = ldap_user_filter self.ldap_user_search_base = ldap_user_search_base self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -102,19 +106,23 @@ def __init__( # Deploy desired state storage account # - This holds the /var/local/ansible container that is mounted by workspaces # - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer - storage_account = WrappedNFSV3StorageAccount( + storage_component = NFSV3StorageAccountComponent( f"{self._name}_storage_account", - account_name=alphanumeric( - f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}" - )[:24], - allowed_ip_addresses=props.admin_ip_addresses, - allowed_service_tag=None, - location=props.location, - resource_group_name=props.resource_group_name, - subnet_id=props.subnet_desired_state_id, + NFSV3StorageAccountProps( + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}" + )[:24], + allowed_ip_addresses=props.admin_ip_addresses, + allowed_service_tag=None, + location=props.location, + log_analytics_workspace=props.log_analytics_workspace, + resource_group_name=props.resource_group_name, + subnet_id=props.subnet_desired_state_id, + ), opts=child_opts, tags=child_tags, ) + storage_account = storage_component.storage_account # Deploy desired state share container_desired_state = NFSV3BlobContainerComponent( f"{self._name}_blob_desired_state", From a1b067fcbb7219ac1a148de0e3ffa70d3dbb8849 Mon Sep 17 00:00:00 2001 From: Matt Craddock Date: Thu, 28 Nov 2024 12:09:50 +0000 Subject: [PATCH 074/100] Update data_safe_haven/commands/users.py Co-authored-by: Jim Madge --- data_safe_haven/commands/users.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index b3a90c0664..661e3e91f0 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -153,8 +153,7 @@ def register( } usernames_to_register = [] for username in usernames: - if username in user_dict.keys(): - user_domain = user_dict[username] + if user_domain:=user_dict.get(username) if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" From 2d148498131bcb68e2226f971bebcfdf6ba6ba9e Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:13:08 +0000 Subject: [PATCH 075/100] add missing colon --- data_safe_haven/commands/users.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 661e3e91f0..80fb1a356b 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -153,7 +153,7 @@ def register( } usernames_to_register = [] for username in usernames: - if user_domain:=user_dict.get(username) + if user_domain := user_dict.get(username): if shm_config.shm.fqdn not in user_domain: logger.error( f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" From 696b192ad1c65dcb9ff4fea922d2ba5628847702 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 14:01:57 +0000 Subject: [PATCH 076/100] Add blob log documentation --- docs/source/management/logs.md | 38 +++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 80852848bd..1b6876794c 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -11,6 +11,38 @@ Select which Log Analytics Workspace you want to view by clicking on the workspa The logs can be filtered using [Kusto Query Language (KQL)](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview). +## Storage logs + +Depending on how different parts of Data Safe Haven storage are provisioned, logs may differ. + +### Sensitive data logs + +The sensitive data containers are the [ingress and egress containers](./data.md). +Logs from these containers are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageBlobLogs` +: Events occurring on the blob containers. +: For example data being uploaded, extracted or read. + +`AzureMetrics` +: Various metrics on blob container utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. + +### Desired state data logs + +The desired state container holds the data necessary to configure virtual machines in an SRE. +Logs from the desired state container are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageBlobLogs` +: Events occurring on the blob containers. +: For example data being uploaded, extracted or read. + +`AzureMetrics` +: Various metrics on blob container utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. + ## Container logs Some of the Data Safe Haven infrastructure is provisioned as containers. @@ -21,7 +53,7 @@ These include, - Gitea and Hedgedoc Logs from all containers are ingested into the [SRE's log analytics workspace](#log-workspace). -There are two logs +There are two tables, `ContainerEvents_CL` : Event logs for the container instance resources such as starting, stopping, crashes and pulling images. @@ -34,7 +66,7 @@ There are two logs Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace) using the [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/azure-monitor-agent-overview). -There are three logs +There are three tables, `Perf` : Usage statistics for individual workspaces, such as percent memory used and percent disk space used. @@ -53,7 +85,7 @@ The firewall plays a critical role in the security of a Data Safe Haven. It filters all outbound traffic through a set of FQDN rules so that each component may only reach necessary and allowed domains. Logs from the firewall are ingested into the [SREs log workspace](#log-workspace). -There are multiple tables, +There are three tables, `AZFWApplicationRule` : Logs from the firewalls FDQN filters. From 04d838faa46e78fd31cc20a4327e22b5041d1807 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 14:55:44 +0000 Subject: [PATCH 077/100] Add diagnostic setting for user data shares --- .../programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/data.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index f69fc9cd45..8cca2dff85 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -211,6 +211,7 @@ def __call__(self) -> None: dns_record=networking.shm_ns_record, dns_server_admin_password=dns.password_admin, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, storage_quota_gb_home=self.config.sre.storage_quota_gb.home, diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 711b76139f..8e1a278770 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,6 +7,7 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, + insights, keyvault, managedidentity, network, @@ -33,6 +34,7 @@ NFSV3BlobContainerProps, SSLCertificate, SSLCertificateProps, + WrappedLogAnalyticsWorkspace, WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -51,6 +53,7 @@ def __init__( dns_record: Input[network.RecordSet], dns_server_admin_password: Input[pulumi_random.RandomPassword], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], storage_quota_gb_home: Input[int], @@ -69,6 +72,7 @@ def __init__( self.dns_record = dns_record self.password_dns_server_admin = dns_server_admin_password self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -615,6 +619,47 @@ def __init__( opts=child_opts, tags=child_tags, ) + # Add diagnostic setting for files + insights.DiagnosticSetting( + f"{storage_account_data_private_user._name}_diagnostic_setting", + name=f"{storage_account_data_private_user._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_private_user.id.apply( + # This is the URI of the fileServices resource which is automatically + # created. + lambda resource_id: resource_id + + "/fileServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) storage.FileShare( f"{storage_account_data_private_user._name}_files_home", access_tier=storage.ShareAccessTier.PREMIUM, From b90a1ff9342f12f665a071fe0b2e04c792b9978d Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 15:03:12 +0000 Subject: [PATCH 078/100] Add diagnostic setting for config file shares --- .../infrastructure/programs/sre/data.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 8e1a278770..0dabd47f71 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -425,6 +425,47 @@ def __init__( resource_group_name=kwargs["resource_group_name"], ) ) + # Add diagnostic setting for files + insights.DiagnosticSetting( + f"{storage_account_data_configuration._name}_diagnostic_setting", + name=f"{storage_account_data_configuration._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_configuration.id.apply( + # This is the URI of the fileService resource which is automatically + # created. + lambda resource_id: resource_id + + "/fileServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) # Set up a private endpoint for the configuration data storage account storage_account_data_configuration_private_endpoint = network.PrivateEndpoint( f"{storage_account_data_configuration._name}_private_endpoint", From 2e56565d77c243b8113a397f0d06e9a5463ebdcc Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:55:55 +0000 Subject: [PATCH 079/100] Add documentation for file share logs --- docs/source/management/logs.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 1b6876794c..408ec9b631 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -43,6 +43,35 @@ There are two tables, : Various metrics on blob container utilisation and performance. : This table is not reserved for the firewall and other resources may log to it. +### User data logs + +The user data file share holds the {ref}`role_researcher`s' [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. +Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageFileLogs` +: NFS events occurring on the file share. +: For example data being written or directories being accessed + +`AzureMetrics` +: Various metrics on file share utilisation and performance. +: This table is not reserved for the user data share and other resources may log to it. + +### Configuration data logs + +There are multiple configuration data file shares. +Each contains the configuration and state data for the Data Safe Haven [services deployed as containers](#container-logs). +Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageFileLogs` +: SMB events occurring on the file share. +: For example data being written or directories being accessed + +`AzureMetrics` +: Various metrics on file share utilisation and performance. +: This table is not reserved for the configuration data shares and other resources may log to it. + ## Container logs Some of the Data Safe Haven infrastructure is provisioned as containers. From 799f96b7944145e40d4c7889bd30f5374c6cf4a9 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:56:55 +0000 Subject: [PATCH 080/100] Correct descriptions --- docs/source/management/logs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 408ec9b631..63314654fc 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -27,7 +27,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the sensitive data containers and other resources may log to it. ### Desired state data logs @@ -41,7 +41,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the desired state data container and other resources may log to it. ### User data logs From 37fdeb7e04d82eb42c4fd08e6b5a83f522692e4e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:56:55 +0000 Subject: [PATCH 081/100] Correct descriptions --- docs/source/management/logs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 1b6876794c..d7607cf98a 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -27,7 +27,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the sensitive data containers and other resources may log to it. ### Desired state data logs @@ -41,7 +41,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the desired state data container and other resources may log to it. ## Container logs From d5802fb893a0d56ab2256229704ab5ce316c1917 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:03:51 +0000 Subject: [PATCH 082/100] Improve reference rendering --- docs/source/management/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 63314654fc..10b9bfb0e5 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -45,7 +45,7 @@ There are two tables, ### User data logs -The user data file share holds the {ref}`role_researcher`s' [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. +The user data file share holds the {ref}`researchers'` [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). There are two tables, From da382e0e600991e4e19728760d5b4e048699a3d1 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:15:59 +0000 Subject: [PATCH 083/100] Use Output concat method Co-authored-by: James Robinson --- .../infrastructure/programs/sre/data.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index e348f25976..a789d61ad0 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -459,12 +459,8 @@ def __init__( }, } ], - resource_uri=storage_account_data_configuration.id.apply( - # This is the URI of the fileService resource which is automatically - # created. - lambda resource_id: resource_id - + "/fileServices/default" - ), + # This is the URI of the automatically created fileService resource + resource_uri=Output.concat(storage_account_data_configuration.id, "/fileServices/default"), workspace_id=props.log_analytics_workspace.id, ) # Set up a private endpoint for the configuration data storage account @@ -700,12 +696,8 @@ def __init__( }, } ], - resource_uri=storage_account_data_private_user.id.apply( - # This is the URI of the fileServices resource which is automatically - # created. - lambda resource_id: resource_id - + "/fileServices/default" - ), + # This is the URI of the automatically created fileService resource + resource_uri=Output.concat(storage_account_data_private_user.id, "/fileServices/default"), workspace_id=props.log_analytics_workspace.id, ) storage.FileShare( From d73704a0742402f638430e6c321e2872f1ff7bb4 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:18:14 +0000 Subject: [PATCH 084/100] Fix linting --- data_safe_haven/infrastructure/programs/sre/data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index a789d61ad0..825861c122 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -460,7 +460,9 @@ def __init__( } ], # This is the URI of the automatically created fileService resource - resource_uri=Output.concat(storage_account_data_configuration.id, "/fileServices/default"), + resource_uri=Output.concat( + storage_account_data_configuration.id, "/fileServices/default" + ), workspace_id=props.log_analytics_workspace.id, ) # Set up a private endpoint for the configuration data storage account @@ -697,7 +699,9 @@ def __init__( } ], # This is the URI of the automatically created fileService resource - resource_uri=Output.concat(storage_account_data_private_user.id, "/fileServices/default"), + resource_uri=Output.concat( + storage_account_data_private_user.id, "/fileServices/default" + ), workspace_id=props.log_analytics_workspace.id, ) storage.FileShare( From 4165cab3824a8c3d8a83f09b631dc86cc27baa67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 03:23:50 +0000 Subject: [PATCH 085/100] Bump the production-dependencies group with 9 updates Bumps the production-dependencies group with 9 updates: | Package | From | To | | --- | --- | --- | | [cryptography](https://github.com/pyca/cryptography) | `43.0.3` | `44.0.0` | | [pulumi-azure-native](https://github.com/pulumi/pulumi-azure-native) | `2.73.1` | `2.74.0` | | [pulumi](https://github.com/pulumi/pulumi) | `3.141.0` | `3.142.0` | | [pydantic](https://github.com/pydantic/pydantic) | `2.10.1` | `2.10.2` | | [pyjwt[crypto]](https://github.com/jpadilla/pyjwt) | `2.10.0` | `2.10.1` | | [typer](https://github.com/fastapi/typer) | `0.13.1` | `0.14.0` | | [pandas-stubs](https://github.com/pandas-dev/pandas-stubs) | `2.2.3.241009` | `2.2.3.241126` | | [ruff](https://github.com/astral-sh/ruff) | `0.8.0` | `0.8.1` | | [pytest](https://github.com/pytest-dev/pytest) | `8.3.3` | `8.3.4` | Updates `cryptography` from 43.0.3 to 44.0.0 - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/43.0.3...44.0.0) Updates `pulumi-azure-native` from 2.73.1 to 2.74.0 - [Release notes](https://github.com/pulumi/pulumi-azure-native/releases) - [Changelog](https://github.com/pulumi/pulumi-azure-native/blob/master/CHANGELOG_OLD.md) - [Commits](https://github.com/pulumi/pulumi-azure-native/compare/v2.73.1...v2.74.0) Updates `pulumi` from 3.141.0 to 3.142.0 - [Release notes](https://github.com/pulumi/pulumi/releases) - [Changelog](https://github.com/pulumi/pulumi/blob/master/CHANGELOG.md) - [Commits](https://github.com/pulumi/pulumi/compare/v3.141.0...v3.142.0) Updates `pydantic` from 2.10.1 to 2.10.2 - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.10.1...v2.10.2) Updates `pyjwt[crypto]` from 2.10.0 to 2.10.1 - [Release notes](https://github.com/jpadilla/pyjwt/releases) - [Changelog](https://github.com/jpadilla/pyjwt/blob/master/CHANGELOG.rst) - [Commits](https://github.com/jpadilla/pyjwt/compare/2.10.0...2.10.1) Updates `typer` from 0.13.1 to 0.14.0 - [Release notes](https://github.com/fastapi/typer/releases) - [Changelog](https://github.com/fastapi/typer/blob/master/docs/release-notes.md) - [Commits](https://github.com/fastapi/typer/compare/0.13.1...0.14.0) Updates `pandas-stubs` from 2.2.3.241009 to 2.2.3.241126 - [Changelog](https://github.com/pandas-dev/pandas-stubs/blob/main/docs/release_procedure.md) - [Commits](https://github.com/pandas-dev/pandas-stubs/compare/v2.2.3.241009...v2.2.3.241126) Updates `ruff` from 0.8.0 to 0.8.1 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.8.0...0.8.1) Updates `pytest` from 8.3.3 to 8.3.4 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.3...8.3.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production update-type: version-update:semver-major dependency-group: production-dependencies - dependency-name: pulumi-azure-native dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pulumi dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: pyjwt[crypto] dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: typer dependency-type: direct:production update-type: version-update:semver-minor dependency-group: production-dependencies - dependency-name: pandas-stubs dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: ruff dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch dependency-group: production-dependencies ... Signed-off-by: dependabot[bot] --- pyproject.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 49099b230d..dc9f8d41dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,20 +42,20 @@ dependencies = [ "azure-storage-file-datalake==12.18.0", "azure-storage-file-share==12.20.0", "chevron==0.14.0", - "cryptography==43.0.3", + "cryptography==44.0.0", "fqdn==1.5.1", "psycopg[binary]==3.1.19", # needed for installation on older MacOS versions - "pulumi-azure-native==2.73.1", + "pulumi-azure-native==2.74.0", "pulumi-azuread==6.0.1", "pulumi-random==4.16.7", - "pulumi==3.141.0", - "pydantic==2.10.1", - "pyjwt[crypto]==2.10.0", + "pulumi==3.142.0", + "pydantic==2.10.2", + "pyjwt[crypto]==2.10.1", "pytz==2024.2", "pyyaml==6.0.2", "rich==13.9.4", "simple-acme-dns==3.2.0", - "typer==0.13.1", + "typer==0.14.0", "websocket-client==1.8.0", ] @@ -77,9 +77,9 @@ lint = [ "ansible==11.0.0", "black==24.10.0", "mypy==1.13.0", - "pandas-stubs==2.2.3.241009", - "pydantic==2.10.1", - "ruff==0.8.0", + "pandas-stubs==2.2.3.241126", + "pydantic==2.10.2", + "ruff==0.8.1", "types-appdirs==1.4.3.5", "types-chevron==0.14.2.20240310", "types-pytz==2024.2.0.20241003", @@ -90,7 +90,7 @@ test = [ "coverage==7.6.8", "freezegun==1.5.1", "pytest-mock==3.14.0", - "pytest==8.3.3", + "pytest==8.3.4", "requests-mock==1.12.1", ] From 04d5802730ca46c9474b87d4d1cb690290e022ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" Date: Mon, 2 Dec 2024 03:30:33 +0000 Subject: [PATCH 086/100] [dependabot skip] :wrench: Update Python requirements files --- .hatch/requirements-lint.txt | 20 ++++++++++---------- .hatch/requirements-test.txt | 32 ++++++++++++++++---------------- .hatch/requirements.txt | 26 +++++++++++++------------- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.hatch/requirements-lint.txt b/.hatch/requirements-lint.txt index e1ab89f54e..c09dc3ebb9 100644 --- a/.hatch/requirements-lint.txt +++ b/.hatch/requirements-lint.txt @@ -5,9 +5,9 @@ # - ansible==11.0.0 # - black==24.10.0 # - mypy==1.13.0 -# - pandas-stubs==2.2.3.241009 -# - pydantic==2.10.1 -# - ruff==0.8.0 +# - pandas-stubs==2.2.3.241126 +# - pydantic==2.10.2 +# - ruff==0.8.1 # - types-appdirs==1.4.3.5 # - types-chevron==0.14.2.20240310 # - types-pytz==2024.2.0.20241003 @@ -81,7 +81,7 @@ click-help-colors==0.9.4 # via molecule colorama==0.4.6 # via tox -cryptography==43.0.3 +cryptography==44.0.0 # via ansible-core distlib==0.3.9 # via @@ -151,7 +151,7 @@ packaging==24.2 # pytest # pytest-ansible # tox -pandas-stubs==2.2.3.241009 +pandas-stubs==2.2.3.241126 # via hatch.envs.lint parsley==1.3 # via bindep @@ -178,7 +178,7 @@ ptyprocess==0.7.0 # via pexpect pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via hatch.envs.lint pydantic-core==2.27.1 # via pydantic @@ -186,7 +186,7 @@ pygments==2.18.0 # via rich pyproject-api==1.8.0 # via tox -pytest==8.3.3 +pytest==8.3.4 # via # pytest-ansible # pytest-xdist @@ -197,7 +197,7 @@ pytest-ansible==24.9.0 # tox-ansible pytest-xdist==3.6.1 # via tox-ansible -python-daemon==3.1.0 +python-daemon==3.1.1 # via ansible-runner python-gnupg==0.5.3 # via ansible-sign @@ -233,7 +233,7 @@ ruamel-yaml==0.18.6 # via ansible-lint ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.1 # via hatch.envs.lint subprocess-tee==0.4.2 # via @@ -265,7 +265,7 @@ tzdata==2024.2 # via ansible-navigator urllib3==2.2.3 # via types-requests -virtualenv==20.27.1 +virtualenv==20.28.0 # via tox wcmatch==10.0 # via diff --git a/.hatch/requirements-test.txt b/.hatch/requirements-test.txt index 8c95d7dce0..ce24e1caee 100644 --- a/.hatch/requirements-test.txt +++ b/.hatch/requirements-test.txt @@ -1,7 +1,7 @@ # # This file is autogenerated by hatch-pip-compile with Python 3.12 # -# [constraints] .hatch/requirements.txt (SHA256: 3586aa93da255077aac182009c06aa28b96ec15387beec4148e3bebd2b9f8852) +# [constraints] .hatch/requirements.txt (SHA256: 9b78097f41c11566a80e32726aefa74a983ac227fce27db9adba04ae7594da1c) # # - appdirs==1.4.4 # - azure-core==1.32.0 @@ -21,25 +21,25 @@ # - azure-storage-file-datalake==12.18.0 # - azure-storage-file-share==12.20.0 # - chevron==0.14.0 -# - cryptography==43.0.3 +# - cryptography==44.0.0 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.73.1 +# - pulumi-azure-native==2.74.0 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.141.0 -# - pydantic==2.10.1 -# - pyjwt[crypto]==2.10.0 +# - pulumi==3.142.0 +# - pydantic==2.10.2 +# - pyjwt[crypto]==2.10.1 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.1 +# - typer==0.14.0 # - websocket-client==1.8.0 # - coverage==7.6.8 # - freezegun==1.5.1 # - pytest-mock==3.14.0 -# - pytest==8.3.3 +# - pytest==8.3.4 # - requests-mock==1.12.1 # @@ -182,7 +182,7 @@ click==8.1.7 # typer coverage==7.6.8 # via hatch.envs.test -cryptography==43.0.3 +cryptography==44.0.0 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -295,14 +295,14 @@ psycopg-binary==3.1.19 # via # -c .hatch/requirements.txt # psycopg -pulumi==3.141.0 +pulumi==3.142.0 # via # -c .hatch/requirements.txt # hatch.envs.test # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.73.1 +pulumi-azure-native==2.74.0 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -318,7 +318,7 @@ pycparser==2.22 # via # -c .hatch/requirements.txt # cffi -pydantic==2.10.1 +pydantic==2.10.2 # via # -c .hatch/requirements.txt # hatch.envs.test @@ -330,12 +330,12 @@ pygments==2.18.0 # via # -c .hatch/requirements.txt # rich -pyjwt==2.10.0 +pyjwt==2.10.1 # via # -c .hatch/requirements.txt # hatch.envs.test # msal -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # -c .hatch/requirements.txt # acme @@ -344,7 +344,7 @@ pyrfc3339==2.0.1 # via # -c .hatch/requirements.txt # acme -pytest==8.3.3 +pytest==8.3.4 # via # hatch.envs.test # pytest-mock @@ -403,7 +403,7 @@ six==1.16.0 # azure-core # pulumi # python-dateutil -typer==0.13.1 +typer==0.14.0 # via # -c .hatch/requirements.txt # hatch.envs.test diff --git a/.hatch/requirements.txt b/.hatch/requirements.txt index 82ad061fc0..f2589f1f68 100644 --- a/.hatch/requirements.txt +++ b/.hatch/requirements.txt @@ -19,20 +19,20 @@ # - azure-storage-file-datalake==12.18.0 # - azure-storage-file-share==12.20.0 # - chevron==0.14.0 -# - cryptography==43.0.3 +# - cryptography==44.0.0 # - fqdn==1.5.1 # - psycopg[binary]==3.1.19 -# - pulumi-azure-native==2.73.1 +# - pulumi-azure-native==2.74.0 # - pulumi-azuread==6.0.1 # - pulumi-random==4.16.7 -# - pulumi==3.141.0 -# - pydantic==2.10.1 -# - pyjwt[crypto]==2.10.0 +# - pulumi==3.142.0 +# - pydantic==2.10.2 +# - pyjwt[crypto]==2.10.1 # - pytz==2024.2 # - pyyaml==6.0.2 # - rich==13.9.4 # - simple-acme-dns==3.2.0 -# - typer==0.13.1 +# - typer==0.14.0 # - websocket-client==1.8.0 # @@ -122,7 +122,7 @@ chevron==0.14.0 # via hatch.envs.default click==8.1.7 # via typer -cryptography==43.0.3 +cryptography==44.0.0 # via # hatch.envs.default # acme @@ -192,13 +192,13 @@ psycopg==3.1.19 # via hatch.envs.default psycopg-binary==3.1.19 # via psycopg -pulumi==3.141.0 +pulumi==3.142.0 # via # hatch.envs.default # pulumi-azure-native # pulumi-azuread # pulumi-random -pulumi-azure-native==2.73.1 +pulumi-azure-native==2.74.0 # via hatch.envs.default pulumi-azuread==6.0.1 # via hatch.envs.default @@ -206,17 +206,17 @@ pulumi-random==4.16.7 # via hatch.envs.default pycparser==2.22 # via cffi -pydantic==2.10.1 +pydantic==2.10.2 # via hatch.envs.default pydantic-core==2.27.1 # via pydantic pygments==2.18.0 # via rich -pyjwt==2.10.0 +pyjwt==2.10.1 # via # hatch.envs.default # msal -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # acme # josepy @@ -257,7 +257,7 @@ six==1.16.0 # via # azure-core # pulumi -typer==0.13.1 +typer==0.14.0 # via hatch.envs.default typing-extensions==4.12.2 # via From 24213888ae05a085980e77e9bcc1f242024a7115 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Dec 2024 03:34:22 +0000 Subject: [PATCH 087/100] Bump karancode/yamllint-github-action from 2.1.1 to 3.0.0 Bumps [karancode/yamllint-github-action](https://github.com/karancode/yamllint-github-action) from 2.1.1 to 3.0.0. - [Release notes](https://github.com/karancode/yamllint-github-action/releases) - [Commits](https://github.com/karancode/yamllint-github-action/compare/v2.1.1...v3.0.0) --- updated-dependencies: - dependency-name: karancode/yamllint-github-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint_code.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint_code.yaml b/.github/workflows/lint_code.yaml index 7786fc4b62..4d0caed16c 100644 --- a/.github/workflows/lint_code.yaml +++ b/.github/workflows/lint_code.yaml @@ -108,7 +108,7 @@ jobs: done rm expanded.tmp - name: Lint YAML - uses: karancode/yamllint-github-action@v2.1.1 + uses: karancode/yamllint-github-action@v3.0.0 with: yamllint_strict: true yamllint_comment: false From c64c7d2bc6828fdc2c338f98940c97488d8d7f93 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Mon, 2 Dec 2024 11:30:40 +0000 Subject: [PATCH 088/100] Split multi-line error message between stdout and log --- data_safe_haven/commands/users.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/data_safe_haven/commands/users.py b/data_safe_haven/commands/users.py index 80fb1a356b..8c8b232ceb 100644 --- a/data_safe_haven/commands/users.py +++ b/data_safe_haven/commands/users.py @@ -5,6 +5,7 @@ import typer +from data_safe_haven import console from data_safe_haven.administration.users import UserHandler from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig from data_safe_haven.exceptions import DataSafeHavenError @@ -155,18 +156,20 @@ def register( for username in usernames: if user_domain := user_dict.get(username): if shm_config.shm.fqdn not in user_domain: + console.print( + f"User [green]'{username}[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" + f"SRE [yellow]'{sre}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue]." + ) logger.error( - f"User [green]'{username}'[/green]'s principal domain name is [blue]'{user_domain}'[/blue].\n" - f"SRE [yellow]'{sre}'[/yellow] belongs to SHM domain [blue]'{shm_config.shm.fqdn}'[/blue].\n" "The user's principal domain name must match the domain of the SRE to be registered." ) else: usernames_to_register.append(username) else: logger.error( - f"Username '{username}' does not belong to this Data Safe Haven deployment.\n" - "Please use 'dsh users add' to create this user." + f"Username '{username}' does not belong to this Data Safe Haven deployment." ) + console.print("Please use 'dsh users add' to create this user.") users.register(sre_config.name, usernames_to_register) except DataSafeHavenError as exc: logger.critical(f"Could not register Data Safe Haven users with SRE '{sre}'.") @@ -270,8 +273,8 @@ def unregister( else: logger.error( f"Username '{username}' does not belong to this Data Safe Haven deployment." - " Please use 'dsh users add' to create it." ) + console.print("Please use 'dsh users add' to create it.") for group_name in ( f"{sre_config.name} Users", f"{sre_config.name} Privileged Users", From ec958a503b9add63b4edd3d20c33b08853fc00bd Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 2 Dec 2024 13:57:47 +0000 Subject: [PATCH 089/100] Update release checklist --- .github/ISSUE_TEMPLATE/release_checklist.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release_checklist.md b/.github/ISSUE_TEMPLATE/release_checklist.md index 575f5c9c53..f4e887e797 100644 --- a/.github/ISSUE_TEMPLATE/release_checklist.md +++ b/.github/ISSUE_TEMPLATE/release_checklist.md @@ -25,11 +25,9 @@ Refer to the [Deployment](https://data-safe-haven.readthedocs.io/en/latest/deplo ### For minor releases and above - [ ] Deploy an SHM from this branch and save a transcript of the deployment logs -- Using the new image, deploy a tier 2 and a tier 3 SRE - - [ ] Save the transcript of your tier 2 SRE deployment - - [ ] Save the transcript of your tier 3 SRE deployment +- [ ] Deploy a tier 2 SRE from this branch and save the transcript of the deployment logs +- [ ] Deploy a tier 3 SRE from this branch and save the transcript of the deployment logs - [ ] Complete the [Security evaluation checklist](https://data-safe-haven.readthedocs.io/en/latest/deployment/security_checklist.html) from the deployment documentation -- [ ] Add the new versions tag as an active build on [Read The Docs](https://readthedocs.org) (You can add as a hidden build, before release, to preview) ### For major releases only From ce17321cd28d2c4157df03a9f2b4a9b7f3ec1b64 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Mon, 2 Dec 2024 14:06:38 +0000 Subject: [PATCH 090/100] Update SECURITY.md --- SECURITY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index c81368a94e..9aee903593 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,8 +7,8 @@ All organisations using an earlier version in production should update to the la | Version | Supported | | --------------------------------------------------------------------------------------- | ------------------ | -| [5.1.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v5.1.0) | :white_check_mark: | -| < 5.1.0 | :x: | +| [5.2.0](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v5.1.0) | :white_check_mark: | +| < 5.2.0 | :x: | ## Reporting a Vulnerability From 9c371ba29523926f8ac2c071e10ee12b3c572d90 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 09:47:20 +0000 Subject: [PATCH 091/100] Correct T2/3 PyPI/CRAN proxy information --- docs/source/overview/sensitivity_tiers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/overview/sensitivity_tiers.md b/docs/source/overview/sensitivity_tiers.md index 4aef9a32fe..995be6ab87 100644 --- a/docs/source/overview/sensitivity_tiers.md +++ b/docs/source/overview/sensitivity_tiers.md @@ -49,7 +49,7 @@ Non-technical restrictions related to information governance procedures may also - connections to the in-browser remote desktop can only be made from an agreed set of IP addresses - outbound connections to the internet from inside the environment are not possible - copy-and-paste between the environment and the user's device is not possible -- access to all packages on PyPI and CRAN is made available through a proxy or mirror server +- access to all packages on PyPI and CRAN is made available through a proxy server Non-technical restrictions related to information governance procedures may also be applied according to your organisation's needs. @@ -63,7 +63,7 @@ At the Turing connections to Tier 2 environments are only permitted from **Organ **Tier 3** environments impose the following technical controls on top of what is required at {ref}`policy_tier_2`. -- a partial replica of agreed PyPI and CRAN packages is made available through a proxy or mirror server +- an agreed subset of PyPI and CRAN packages is made available through a proxy server Non-technical restrictions related to information governance procedures may also be applied according to your organisation's needs. From ed0ae2bb7293b507325712b455d080baf47faef6 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:49:11 +0000 Subject: [PATCH 092/100] exclude security_checklist_template --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index f262d36dc2..fa99142138 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,7 +64,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["**/*.partial.md"] +exclude_patterns = ["**/*.partial.md", "**/security_checklist_template.md"] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for From 97fee49ed913c548eb557d003e7dae5380b922b6 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:51:02 +0000 Subject: [PATCH 093/100] Add download link for security checklist --- docs/source/deployment/security_checklist.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md index 7c6036402a..d30e9c3baf 100644 --- a/docs/source/deployment/security_checklist.md +++ b/docs/source/deployment/security_checklist.md @@ -8,6 +8,7 @@ Organisations are responsible for making their own decisions about the suitabili ``` In this check list we aim to evaluate our deployment against the {ref}`security configuration ` that we apply at the Alan Turing Institute. +A copy of this template in Markdown format is {download}`available for download `. The security checklist currently focuses on checks that can evaluate these security requirements for {ref}`policy_tier_2` (or greater) SREs (with some steps noted as specific to a tier): ## How to use this checklist From af232b7b59205d09d998ddc067d59f1f1d7d2748 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:51:19 +0000 Subject: [PATCH 094/100] add checklist_template.md --- .../security_checklist_template.md | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 docs/source/deployment/security_checklist/security_checklist_template.md diff --git a/docs/source/deployment/security_checklist/security_checklist_template.md b/docs/source/deployment/security_checklist/security_checklist_template.md new file mode 100644 index 0000000000..4c7489557c --- /dev/null +++ b/docs/source/deployment/security_checklist/security_checklist_template.md @@ -0,0 +1,128 @@ +# Security checklist +Running on SHM/SREs deployed using commit xxxxxx + +## Summary ++ :white_check_mark: x tests passed +- :partly_sunny: x tests partially passed (see below for more details) +- :fast_forward: x tests skipped (see below for more details) +- :x: x tests failed (see below for more details) + +## Details +- Any additional details as referred to in the summary + +### Multifactor Authentication and Password strength + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Users can reset their own password ++ Verify that: User can reset their own password + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: non-registered users cannot connect to any SRE workspace + + Verify that: User can authenticate but cannot see any workspaces + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: registered users can see SRE workspaces + + Verify that: User can authenticate and can see workspaces + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Authenticated user can access workspaces + + Verify that: You can connect to any workspace + + +### Isolated Network ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from a workspace + + Verify that: Browsing to the service fails + + + Verify that: You cannot access the service using curl + + + Verify: You cannot get the IP address for the service using nslookup + + + +### User devices +#### Tier 2: ++ Connect to the environment using an allowed IP address and credentials + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds ++ Connect to the environment from an IP address that is not allowed but with correct credentials + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + +#### Tier 3: ++ All managed devices should be provided by a known IT team at an approved organisation. + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. ++ Connect to the environment using an allowed IP address and credentials + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds ++ Connect to the environment from an IP address that is not allowed but with correct credentials + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + +#### Tiers 2+: ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses + + In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway + + Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only + + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: all other NSGs have an inbound Deny All rule and no higher priority rule allowing inbound connections from outside the Virtual Network + +### Physical security + +#### Tier 3 only + ++ Attempt to connect to the Tier 3 SRE web client from home using a managed device and the correct VPN connection and credentials. + + :fast_forward: Verify that: connection fails. ++ Attempt to connect from research office using a managed device and the correct VPN connection and credentials. + + :fast_forward: Verify that: connection succeeds + + :fast_forward: Verify that: the network IP ranges corresponding to the research spaces correspond to those allowed by storage account firewall + + :fast_forward: Verify that: physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high + +### Remote connections + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH + + Verify that: SSH login by fully-qualified domain name fails + + + Verify that: SSH login by public IP address fails + + ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the remote desktop web client application gateway (shm--sre--ag-entrypoint) and the firewall are the only SRE resources with public IP addresses. + +### Copy-and-paste ++ Unable to paste text from a local device into a workspace + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails ++ Unable to copy text from a workspace to a local device + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails + +### Data ingress ++ Check that the **System Manager** can send an upload token to the **Dataset Provider Representative** + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. ++ Ensure that data ingress works only for connections from the accepted IP address range + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address ++ Check that the upload fails if the token has expired + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) + +### Data egress ++ Confirm that a non-privileged user is able to read the different storage volumes and write to output + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide ++ Confirm that System Manager can see and download files from output + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. + + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download + +### Software package repositories + +#### Tier 2: ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages + + Verify that: pytz can be installed + + + + Verify that: awscli can be installed + + + +#### Tier 3: ++ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages + + Verify: pytz can be installed + + + + Verify: awscli cannot be installed + From 97fe53a40cdd5b446941b16948b3f1de6b7dfbd1 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 3 Dec 2024 10:33:15 +0000 Subject: [PATCH 095/100] Add checklist template --- docs/source/conf.py | 5 +- docs/source/deployment/security_checklist.md | 2 + .../security_checklist_template.md | 163 ++++++++++++++++++ 3 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 docs/source/deployment/security_checklist/security_checklist_template.md diff --git a/docs/source/conf.py b/docs/source/conf.py index f262d36dc2..dcc77557e7 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -64,7 +64,10 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["**/*.partial.md"] +exclude_patterns = [ + "**/*.partial.md", + "deployment/security_checklist/security_checklist_template.md", +] # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md index 7c6036402a..2c4ca4a6ca 100644 --- a/docs/source/deployment/security_checklist.md +++ b/docs/source/deployment/security_checklist.md @@ -559,3 +559,5 @@ To minimise the risk of unauthorised access to the dataset while the ingress vol ``` ```` + +{download}`this file <./security_checklist/security_checklist_template.md>`. diff --git a/docs/source/deployment/security_checklist/security_checklist_template.md b/docs/source/deployment/security_checklist/security_checklist_template.md new file mode 100644 index 0000000000..ba762aa699 --- /dev/null +++ b/docs/source/deployment/security_checklist/security_checklist_template.md @@ -0,0 +1,163 @@ +# Security checklist + +Running on SHM/SREs deployed using commit XXXXXXX + +## Summary + +- :white_check_mark: N tests passed +- :partly_sunny: N tests partially passed (see below for more details) +- :fast_forward: N tests skipped (see below for more details) +- :x: N tests failed (see below for more details) + +## Details + +Some security checks were skipped since: + +- No managed device was available +- No access to a physical space with its own dedicated network was possible + +### Multifactor Authentication and Password strength + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the SRE standard user cannot access the apps + -
:camera: Verify before adding to group: Microsoft Remote Desktop: Login works but apps cannot be viewed + +
+ -
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA + +
+ +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that adding the **SRE standard user** to the SRE group on the domain controller does not give them access + -
:camera: Verify after adding to group: Microsoft Remote Desktop: Login works and apps can be viewed + +
+ -
:camera: Verify after adding to group: Microsoft Remote Desktop: attempt to login to DSVM Main (Desktop) fails + +
+ -
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA + +
+ +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** is able to successfully set up MFA + -
:camera: Verify: successfully set up MFA + +
+ +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can authenticate with MFA + -
:camera: Verify: Guacamole: respond to the MFA prompt + 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> +
+ -
:camera: Verify: Microsoft Remote Desktop: attempt to log in to DSVM Main (Desktop) and respond to the MFA prompt + 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> +
+ +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can access the DSVM desktop + -
:camera: Verify: Microsoft Remote Desktop: connect to DSVM Main (Desktop) + +
+ -
:camera: Verify: Guacamole: connect to Desktop: Ubuntu0 + +
+ +### Isolated Network + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connect to the SHM DC and NPS if connected to the SHM VPN +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the SHM DC and NPS if not connected to the SHM VPN +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from within a DSVM on the SRE network. + -
:camera: Verify: Connection fails + 122045859-8142bb00-cdde-11eb-920c-3a162a180647.png"> +
+ -
:camera: Verify: that you cannot access a website using curl + +
+ -
:camera: Verify: that you cannot get the IP address for a website using nslookup + +
+- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that users cannot connect between two SREs within the same SHM, even if they have access to both SREs + -
:camera: Verify: SSH connection fails + +
+- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules are set appropriately to block outgoing traffic + -
:camera: Verify: access rules + +
+ +### User devices + +#### Tier 2: + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection + +#### Tier 3: + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check user lacks root access +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection with an allow-listed IP address + +#### Tiers 2+: + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses + -
:camera: Verify: access rules + +
+- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: All non-deployment NSGs have rules denying inbound connections from outside the Virtual Network + +### Physical security + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from outside was not tested +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from inside was not tested +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check the network IP ranges corresponding to the research spaces and compare against the IPs accepted by the firewall. +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so confirmation of physical measures was not tested + +### Remote connections + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH + -
:camera: Verify: SSH connection by FQDN fails + +
+ -
:camera: Verify: SSH connection by public IP address fails + +
+- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: The remote desktop server is the only SRE resource with a public IP address + +### Copy-and-paste + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to paste local text into a DSVM +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to copy text from a DSVM +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Copy between VMs in an SRE succeeds + +### Data ingress + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** secure upload token successfully created with write-only permissions +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** token was sent using a secure, out-of-band communication channel (e.g. secure email) +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an allow-listed IP address succeeds +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** downloading a file from an allow-listed IP address fails +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an non-allowed IP address fails +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection during lifetime of short-duration token succeeds +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection after lifetime of short-duration token fails +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading different file types succeeds + +### Storage volumes and egress + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/output` volume +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can only read from the `/data` volume +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to their directory in `/home` +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/shared` volume +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can see the files ready for egress +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can download egress-ready files + +### Package mirrors + +#### Tier 2: + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages + -
:camera: Verify: botocore can be installed + +
+ +#### Tier 3: + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages + -
:camera: Verify: aero-calc can be installed; botocore cannot be installed + +
From 6556e44443b3135bb2d3a4ecdea9e3b115a81cf2 Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:42:24 +0000 Subject: [PATCH 096/100] fixing markdown linting --- .../security_checklist_template.md | 146 ++++++++++-------- 1 file changed, 78 insertions(+), 68 deletions(-) diff --git a/docs/source/deployment/security_checklist/security_checklist_template.md b/docs/source/deployment/security_checklist/security_checklist_template.md index 4c7489557c..bcb8a9931b 100644 --- a/docs/source/deployment/security_checklist/security_checklist_template.md +++ b/docs/source/deployment/security_checklist/security_checklist_template.md @@ -2,127 +2,137 @@ Running on SHM/SREs deployed using commit xxxxxx ## Summary -+ :white_check_mark: x tests passed + +- :white_check_mark: x tests passed - :partly_sunny: x tests partially passed (see below for more details) - :fast_forward: x tests skipped (see below for more details) - :x: x tests failed (see below for more details) ## Details + - Any additional details as referred to in the summary ### Multifactor Authentication and Password strength -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Users can reset their own password -+ Verify that: User can reset their own password +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Users can reset their own password +- Verify that: User can reset their own password -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: non-registered users cannot connect to any SRE workspace - + Verify that: User can authenticate but cannot see any workspaces +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: non-registered users cannot connect to any SRE workspace + - Verify that: User can authenticate but cannot see any workspaces -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: registered users can see SRE workspaces - + Verify that: User can authenticate and can see workspaces +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: registered users can see SRE workspaces + - Verify that: User can authenticate and can see workspaces -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Authenticated user can access workspaces - + Verify that: You can connect to any workspace +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Authenticated user can access workspaces + - Verify that: You can connect to any workspace ### Isolated Network -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from a workspace - + Verify that: Browsing to the service fails + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from a workspace + - Verify that: Browsing to the service fails - + Verify that: You cannot access the service using curl + - Verify that: You cannot access the service using curl - + Verify: You cannot get the IP address for the service using nslookup + - Verify: You cannot get the IP address for the service using nslookup - ### User devices + #### Tier 2: -+ Connect to the environment using an allowed IP address and credentials - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds -+ Connect to the environment from an IP address that is not allowed but with correct credentials - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + +- Connect to the environment using an allowed IP address and credentials + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds +- Connect to the environment from an IP address that is not allowed but with correct credentials + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails #### Tier 3: -+ All managed devices should be provided by a known IT team at an approved organisation. - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. -+ Connect to the environment using an allowed IP address and credentials - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds -+ Connect to the environment from an IP address that is not allowed but with correct credentials - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + +- All managed devices should be provided by a known IT team at an approved organisation. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. +- Connect to the environment using an allowed IP address and credentials + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds +- Connect to the environment from an IP address that is not allowed but with correct credentials + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails #### Tiers 2+: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses - + In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway - + Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses + - In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway + - Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: all other NSGs have an inbound Deny All rule and no higher priority rule allowing inbound connections from outside the Virtual Network +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: all other NSGs have an inbound Deny All rule and no higher priority rule allowing inbound connections from outside the Virtual Network ### Physical security #### Tier 3 only -+ Attempt to connect to the Tier 3 SRE web client from home using a managed device and the correct VPN connection and credentials. - + :fast_forward: Verify that: connection fails. -+ Attempt to connect from research office using a managed device and the correct VPN connection and credentials. - + :fast_forward: Verify that: connection succeeds - + :fast_forward: Verify that: the network IP ranges corresponding to the research spaces correspond to those allowed by storage account firewall - + :fast_forward: Verify that: physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high +- Attempt to connect to the Tier 3 SRE web client from home using a managed device and the correct VPN connection and credentials. + - :fast_forward: Verify that: connection fails. +- Attempt to connect from research office using a managed device and the correct VPN connection and credentials. + - :fast_forward: Verify that: connection succeeds + - :fast_forward: Verify that: the network IP ranges corresponding to the research spaces correspond to those allowed by storage account firewall + - :fast_forward: Verify that: physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high ### Remote connections -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH - + Verify that: SSH login by fully-qualified domain name fails +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH + - Verify that: SSH login by fully-qualified domain name fails - + Verify that: SSH login by public IP address fails + - Verify that: SSH login by public IP address fails -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the remote desktop web client application gateway (shm--sre--ag-entrypoint) and the firewall are the only SRE resources with public IP addresses. +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the remote desktop web client application gateway (shm--sre--ag-entrypoint) and the firewall are the only SRE resources with public IP addresses. ### Copy-and-paste -+ Unable to paste text from a local device into a workspace - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails -+ Unable to copy text from a workspace to a local device - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails + +- Unable to paste text from a local device into a workspace + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails +- Unable to copy text from a workspace to a local device + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails ### Data ingress -+ Check that the **System Manager** can send an upload token to the **Dataset Provider Representative** - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. -+ Ensure that data ingress works only for connections from the accepted IP address range - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address -+ Check that the upload fails if the token has expired - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) + +- Check that the **System Manager** can send an upload token to the **Dataset Provider Representative** + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. +- Ensure that data ingress works only for connections from the accepted IP address range + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address +- Check that the upload fails if the token has expired + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) ### Data egress -+ Confirm that a non-privileged user is able to read the different storage volumes and write to output - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide -+ Confirm that System Manager can see and download files from output - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. - + :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download + +- Confirm that a non-privileged user is able to read the different storage volumes and write to output + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide +- Confirm that System Manager can see and download files from output + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download ### Software package repositories #### Tier 2: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages - + Verify that: pytz can be installed - - + Verify that: awscli can be installed +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages + - Verify that: pytz can be installed + - Verify that: awscli can be installed + #### Tier 3: -+ :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages - + Verify: pytz can be installed + +- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages + - Verify: pytz can be installed - + Verify: awscli cannot be installed + - Verify: awscli cannot be installed From 508a77818b3e56b74ece8c48479593987475634d Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:44:03 +0000 Subject: [PATCH 097/100] more linting --- .../security_checklist_template.md | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/docs/source/deployment/security_checklist/security_checklist_template.md b/docs/source/deployment/security_checklist/security_checklist_template.md index bcb8a9931b..55c2d3b543 100644 --- a/docs/source/deployment/security_checklist/security_checklist_template.md +++ b/docs/source/deployment/security_checklist/security_checklist_template.md @@ -1,4 +1,5 @@ # Security checklist + Running on SHM/SREs deployed using commit xxxxxx ## Summary @@ -18,23 +19,23 @@ Running on SHM/SREs deployed using commit xxxxxx - Verify that: User can reset their own password - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: non-registered users cannot connect to any SRE workspace - - Verify that: User can authenticate but cannot see any workspaces + - Verify that: User can authenticate but cannot see any workspaces - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: registered users can see SRE workspaces - - Verify that: User can authenticate and can see workspaces + - Verify that: User can authenticate and can see workspaces - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Authenticated user can access workspaces - - Verify that: You can connect to any workspace + - Verify that: You can connect to any workspace ### Isolated Network - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from a workspace - - Verify that: Browsing to the service fails + - Verify that: Browsing to the service fails - - Verify that: You cannot access the service using curl + - Verify that: You cannot access the service using curl - - Verify: You cannot get the IP address for the service using nslookup + - Verify: You cannot get the IP address for the service using nslookup ### User devices @@ -42,26 +43,26 @@ Running on SHM/SREs deployed using commit xxxxxx #### Tier 2: - Connect to the environment using an allowed IP address and credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds - Connect to the environment from an IP address that is not allowed but with correct credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails #### Tier 3: - All managed devices should be provided by a known IT team at an approved organisation. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. - Connect to the environment using an allowed IP address and credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds - Connect to the environment from an IP address that is not allowed but with correct credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails #### Tiers 2+: - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses - - In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway - - Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only + - In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway + - Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: all other NSGs have an inbound Deny All rule and no higher priority rule allowing inbound connections from outside the Virtual Network @@ -80,9 +81,9 @@ Running on SHM/SREs deployed using commit xxxxxx ### Remote connections - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH - - Verify that: SSH login by fully-qualified domain name fails + - Verify that: SSH login by fully-qualified domain name fails - - Verify that: SSH login by public IP address fails + - Verify that: SSH login by public IP address fails - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the remote desktop web client application gateway (shm--sre--ag-entrypoint) and the firewall are the only SRE resources with public IP addresses. @@ -97,42 +98,42 @@ Running on SHM/SREs deployed using commit xxxxxx ### Data ingress - Check that the **System Manager** can send an upload token to the **Dataset Provider Representative** - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. - Ensure that data ingress works only for connections from the accepted IP address range - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address - Check that the upload fails if the token has expired - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) ### Data egress - Confirm that a non-privileged user is able to read the different storage volumes and write to output - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide - Confirm that System Manager can see and download files from output - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download ### Software package repositories #### Tier 2: - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages - - Verify that: pytz can be installed + - Verify that: pytz can be installed - - Verify that: awscli can be installed + - Verify that: awscli can be installed #### Tier 3: - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages - - Verify: pytz can be installed + - Verify: pytz can be installed - - Verify: awscli cannot be installed + - Verify: awscli cannot be installed From a9c7815ebb57e1b41947b327ac8561b2b4eb0e9c Mon Sep 17 00:00:00 2001 From: Matt Craddock <5796417+craddm@users.noreply.github.com> Date: Tue, 3 Dec 2024 12:49:16 +0000 Subject: [PATCH 098/100] more linting --- .../security_checklist_template.md | 82 +++++++++---------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/docs/source/deployment/security_checklist/security_checklist_template.md b/docs/source/deployment/security_checklist/security_checklist_template.md index 55c2d3b543..b8069f839b 100644 --- a/docs/source/deployment/security_checklist/security_checklist_template.md +++ b/docs/source/deployment/security_checklist/security_checklist_template.md @@ -16,26 +16,26 @@ Running on SHM/SREs deployed using commit xxxxxx ### Multifactor Authentication and Password strength - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Users can reset their own password -- Verify that: User can reset their own password + - Verify that: User can reset their own password - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: non-registered users cannot connect to any SRE workspace - - Verify that: User can authenticate but cannot see any workspaces + - Verify that: User can authenticate but cannot see any workspaces - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: registered users can see SRE workspaces - - Verify that: User can authenticate and can see workspaces + - Verify that: User can authenticate and can see workspaces - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check: Authenticated user can access workspaces - - Verify that: You can connect to any workspace + - Verify that: You can connect to any workspace ### Isolated Network - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from a workspace - - Verify that: Browsing to the service fails + - Verify that: Browsing to the service fails - - Verify that: You cannot access the service using curl + - Verify that: You cannot access the service using curl - - Verify: You cannot get the IP address for the service using nslookup + - Verify: You cannot get the IP address for the service using nslookup ### User devices @@ -43,26 +43,26 @@ Running on SHM/SREs deployed using commit xxxxxx #### Tier 2: - Connect to the environment using an allowed IP address and credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds - Connect to the environment from an IP address that is not allowed but with correct credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails #### Tier 3: - All managed devices should be provided by a known IT team at an approved organisation. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the IT team of the approved organisation take responsibility for managing the device. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the user does not have administrator permissions on the device. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: allowed IP addresses are exclusive to managed devices. - Connect to the environment using an allowed IP address and credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection succeeds - Connect to the environment from an IP address that is not allowed but with correct credentials - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: Connection fails #### Tiers 2+: - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses - - In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway - - Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only + - In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway + - Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: all other NSGs have an inbound Deny All rule and no higher priority rule allowing inbound connections from outside the Virtual Network @@ -72,18 +72,18 @@ Running on SHM/SREs deployed using commit xxxxxx #### Tier 3 only - Attempt to connect to the Tier 3 SRE web client from home using a managed device and the correct VPN connection and credentials. - - :fast_forward: Verify that: connection fails. + - :fast_forward: Verify that: connection fails. - Attempt to connect from research office using a managed device and the correct VPN connection and credentials. - - :fast_forward: Verify that: connection succeeds - - :fast_forward: Verify that: the network IP ranges corresponding to the research spaces correspond to those allowed by storage account firewall - - :fast_forward: Verify that: physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high + - :fast_forward: Verify that: connection succeeds + - :fast_forward: Verify that: the network IP ranges corresponding to the research spaces correspond to those allowed by storage account firewall + - :fast_forward: Verify that: physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high ### Remote connections - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH - - Verify that: SSH login by fully-qualified domain name fails + - Verify that: SSH login by fully-qualified domain name fails - - Verify that: SSH login by public IP address fails + - Verify that: SSH login by public IP address fails - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the remote desktop web client application gateway (shm--sre--ag-entrypoint) and the firewall are the only SRE resources with public IP addresses. @@ -91,49 +91,47 @@ Running on SHM/SREs deployed using commit xxxxxx ### Copy-and-paste - Unable to paste text from a local device into a workspace - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails - Unable to copy text from a workspace to a local device - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: paste fails ### Data ingress - Check that the **System Manager** can send an upload token to the **Dataset Provider Representative** - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the upload token is successfully created. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you are able to send this token using a secure mechanism. - Ensure that data ingress works only for connections from the accepted IP address range - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: writing succeeds by uploading a file + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the access token fails when using a device with a non-allowed IP address - Check that the upload fails if the token has expired - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can connect and write with the token during the duration + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you cannot connect and write with the token after the duration has expired + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) ### Data egress - Confirm that a non-privileged user is able to read the different storage volumes and write to output - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the `/mnt/output` volume exists and can be written to + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: the permissions of other storage volumes match that described in the user guide - Confirm that System Manager can see and download files from output - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. - - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: you can see the files written to the `/mnt/output` storage volume. + - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Verify that: a written file can be taken out of the environment via download ### Software package repositories #### Tier 2: - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages - - Verify that: pytz can be installed + - Verify that: pytz can be installed - - - Verify that: awscli can be installed + - Verify that: awscli can be installed #### Tier 3: - :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages - - Verify: pytz can be installed + - Verify: pytz can be installed - - - Verify: awscli cannot be installed + - Verify: awscli cannot be installed From 4409e5cd88ac7dcaf39098f41e9ccb9c31b9776f Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 3 Dec 2024 11:31:48 +0000 Subject: [PATCH 099/100] Update checklist template Co-authored-by: Matt Craddock <5796417+craddm@users.noreply.github.com> --- docs/source/deployment/security_checklist.md | 4 +- .../security_checklist_template.md | 232 ++++++++---------- 2 files changed, 106 insertions(+), 130 deletions(-) diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md index 2c4ca4a6ca..b2f8308181 100644 --- a/docs/source/deployment/security_checklist.md +++ b/docs/source/deployment/security_checklist.md @@ -20,6 +20,8 @@ Work your way through the actions described in each section, taking care to noti - {{white_check_mark}} This indicates a checklist item for which a screenshot is either not appropriate or difficult ``` +You can use {download}`this template Markdown file <./security_checklist/security_checklist_template.md>` to complete the checklist. + ## Prerequisites ### Roles @@ -559,5 +561,3 @@ To minimise the risk of unauthorised access to the dataset while the ingress vol ``` ```` - -{download}`this file <./security_checklist/security_checklist_template.md>`. diff --git a/docs/source/deployment/security_checklist/security_checklist_template.md b/docs/source/deployment/security_checklist/security_checklist_template.md index ba762aa699..5c1a64a119 100644 --- a/docs/source/deployment/security_checklist/security_checklist_template.md +++ b/docs/source/deployment/security_checklist/security_checklist_template.md @@ -1,163 +1,139 @@ # Security checklist -Running on SHM/SREs deployed using commit XXXXXXX +Running on SHM/SREs deployed using commit ## Summary -- :white_check_mark: N tests passed -- :partly_sunny: N tests partially passed (see below for more details) -- :fast_forward: N tests skipped (see below for more details) -- :x: N tests failed (see below for more details) +- :white_check_mark: tests passed +- :partly_sunny: tests partially passed (see below for more details) +- :fast_forward: tests skipped (see below for more details) +- :x: tests failed (see below for more details) ## Details -Some security checks were skipped since: +Some security checks were skipped because: -- No managed device was available -- No access to a physical space with its own dedicated network was possible +- … +- … ### Multifactor Authentication and Password strength -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the SRE standard user cannot access the apps - -
:camera: Verify before adding to group: Microsoft Remote Desktop: Login works but apps cannot be viewed - -
- -
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA - -
- -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that adding the **SRE standard user** to the SRE group on the domain controller does not give them access - -
:camera: Verify after adding to group: Microsoft Remote Desktop: Login works and apps can be viewed - -
- -
:camera: Verify after adding to group: Microsoft Remote Desktop: attempt to login to DSVM Main (Desktop) fails - -
- -
:camera: Verify before adding to group: Guacamole: User is prompted to setup MFA - -
- -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** is able to successfully set up MFA - -
:camera: Verify: successfully set up MFA - -
- -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can authenticate with MFA - -
:camera: Verify: Guacamole: respond to the MFA prompt - 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> -
- -
:camera: Verify: Microsoft Remote Desktop: attempt to log in to DSVM Main (Desktop) and respond to the MFA prompt - 122043131-47bc8080-cddb-11eb-8578-e45ab3efaef0.png"> -
- -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that the **SRE standard user** can access the DSVM desktop - -
:camera: Verify: Microsoft Remote Desktop: connect to DSVM Main (Desktop) - -
- -
:camera: Verify: Guacamole: connect to Desktop: Ubuntu0 - -
+- :white_check_mark: Check: Users can reset their own password +- Verify that: User can reset their own password + + +- :white_check_mark: Check: non-registered users cannot connect to any SRE workspace + - Verify that: User can authenticate but cannot see any workspaces + +- :white_check_mark: Check: registered users can see SRE workspaces + - Verify that: User can authenticate and can see workspaces + +- :white_check_mark: Check: Authenticated user can access workspaces + - Verify that: You can connect to any workspace + ### Isolated Network -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connect to the SHM DC and NPS if connected to the SHM VPN -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the SHM DC and NPS if not connected to the SHM VPN -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Fail to connect to the internet from within a DSVM on the SRE network. - -
:camera: Verify: Connection fails - 122045859-8142bb00-cdde-11eb-920c-3a162a180647.png"> -
- -
:camera: Verify: that you cannot access a website using curl - -
- -
:camera: Verify: that you cannot get the IP address for a website using nslookup - -
-- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check that users cannot connect between two SREs within the same SHM, even if they have access to both SREs - -
:camera: Verify: SSH connection fails - -
-- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules are set appropriately to block outgoing traffic - -
:camera: Verify: access rules - -
+- :white_check_mark: Fail to connect to the internet from a workspace + - Verify that: Browsing to the service fails + + - Verify that: You cannot access the service using curl + + - Verify: You cannot get the IP address for the service using nslookup + ### User devices -#### Tier 2: +#### Tier 2 -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection +- Connect to the environment using an allowed IP address and credentials + - :white_check_mark: Verify that: Connection succeeds +- Connect to the environment from an IP address that is not allowed but with correct credentials + - :white_check_mark: Verify that: Connection fails -#### Tier 3: +#### Tier 3 -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check user lacks root access -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Connection succeeds from a personal device with an allow-listed IP address -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No managed device available to check connection with an allow-listed IP address +- All managed devices should be provided by a known IT team at an approved organisation. + - :fast_forward: Verify that: the IT team of the approved organisation take responsibility for managing the device. + - :fast_forward: Verify that: the user does not have administrator permissions on the device. + - :fast_forward: Verify that: allowed IP addresses are exclusive to managed devices. +- Connect to the environment using an allowed IP address and credentials + - :fast_forward: Verify that: Connection succeeds +- Connect to the environment from an IP address that is not allowed but with correct credentials + - :fast_forward: Verify that: Connection fails -#### Tiers 2+: +#### Tiers 2 and above -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Network rules permit access only from allow-listed IP addresses - -
:camera: Verify: access rules - -
-- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: All non-deployment NSGs have rules denying inbound connections from outside the Virtual Network +- :white_check_mark: Network rules permit access only from allow-listed IP addresses + - In the Azure portal navigate to the Guacamole application gateway NSG for this SRE shm--sre--nsg-application-gateway + - Verify that: the NSG has network rules allowing Inbound access from allowed IP addresses only + +- :white_check_mark: all other NSGs have an inbound Deny All rule and no higher priority rule allowing inbound connections from outside the Virtual Network ### Physical security -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from outside was not tested -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so connection from inside was not tested -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Check the network IP ranges corresponding to the research spaces and compare against the IPs accepted by the firewall. -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: No secure physical space available so confirmation of physical measures was not tested +#### Tier 3 only + +- Attempt to connect to the Tier 3 SRE web client from home using a managed device and the correct VPN connection and credentials. + - :fast_forward: Verify that: connection fails. +- Attempt to connect from research office using a managed device and the correct VPN connection and credentials. + - :fast_forward: Verify that: connection succeeds + - :fast_forward: Verify that: the network IP ranges corresponding to the research spaces correspond to those allowed by storage account firewall + - :fast_forward: Verify that: physical measures such as screen adaptions or desk partitions are present if risk of visual eavesdropping is high ### Remote connections -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to connect as a user to the remote desktop server via SSH - -
:camera: Verify: SSH connection by FQDN fails - -
- -
:camera: Verify: SSH connection by public IP address fails - -
-- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: The remote desktop server is the only SRE resource with a public IP address +- :white_check_mark: Unable to connect as a user to the remote desktop server via SSH + - Verify that: SSH login by fully-qualified domain name fails + + - Verify that: SSH login by public IP address fails + +- :white_check_mark: Verify that: the remote desktop web client application gateway (shm--sre--ag-entrypoint) and the firewall are the only SRE resources with public IP addresses. ### Copy-and-paste -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to paste local text into a DSVM -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Unable to copy text from a DSVM -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Copy between VMs in an SRE succeeds +- Unable to paste text from a local device into a workspace + - :white_check_mark: Verify that: paste fails +- Unable to copy text from a workspace to a local device + - :white_check_mark: Verify that: paste fails ### Data ingress -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** secure upload token successfully created with write-only permissions -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** token was sent using a secure, out-of-band communication channel (e.g. secure email) -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an allow-listed IP address succeeds -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** downloading a file from an allow-listed IP address fails -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading a file from an non-allowed IP address fails -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection during lifetime of short-duration token succeeds -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** connection after lifetime of short-duration token fails -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **Data Provider:** uploading different file types succeeds - -### Storage volumes and egress - -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/output` volume -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can only read from the `/data` volume -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to their directory in `/home` -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **SRE standard user** can read and write to the `/shared` volume -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can see the files ready for egress -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: **System administrator:** can download egress-ready files - -### Package mirrors - -#### Tier 2: - -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install any packages - -
:camera: Verify: botocore can be installed - -
- -#### Tier 3: - -- :white_check_mark:/:partly_sunny:/:fast_forward:/:x: Can install only allow-listed packages - -
:camera: Verify: aero-calc can be installed; botocore cannot be installed - -
+- Check that the **System Manager** can send an upload token to the **Dataset Provider Representative** + - :white_check_mark: Verify that: the upload token is successfully created. + - :white_check_mark: Verify that: you are able to send this token using a secure mechanism. +- Ensure that data ingress works only for connections from the accepted IP address range + - :white_check_mark: Verify that: writing succeeds by uploading a file + - :white_check_mark: Verify that: attempting to open or download any of the files results in the following error: "Failed to start transfer: Insufficient credentials" under the Activities pane at the bottom of the MS Azure Storage Explorer window. + - :white_check_mark: Verify that: the access token fails when using a device with a non-allowed IP address +- Check that the upload fails if the token has expired + - :white_check_mark: Verify that: you can connect and write with the token during the duration + - :white_check_mark: Verify that: you cannot connect and write with the token after the duration has expired + - :white_check_mark: Verify that:the data ingress process works by uploading different kinds of files, e.g. data, images, scripts (if appropriate) + +### Data egress + +- Confirm that a non-privileged user is able to read the different storage volumes and write to output + - :white_check_mark: Verify that: the `/mnt/output` volume exists and can be written to + - :white_check_mark: Verify that: the permissions of other storage volumes match that described in the user guide +- Confirm that System Manager can see and download files from output + - :white_check_mark: Verify that: you can see the files written to the `/mnt/output` storage volume. + - :white_check_mark: Verify that: a written file can be taken out of the environment via download + +### Software package repositories + +#### Tier 2 + +- :white_check_mark: Can install any packages + - Verify that: pytz can be installed + + - Verify that: awscli can be installed + + +#### Tier 3 + +- :white_check_mark: Can install only allow-listed packages + - Verify: pytz can be installed + + - Verify: awscli cannot be installed + From bebeea64002ba45e945ac38a5ae93fce395559e2 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 5 Dec 2024 12:09:05 +0000 Subject: [PATCH 100/100] Correct docstring --- .../infrastructure/programs/sre/software_repositories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/software_repositories.py b/data_safe_haven/infrastructure/programs/sre/software_repositories.py index 420ca5c5a2..be67c3e8af 100644 --- a/data_safe_haven/infrastructure/programs/sre/software_repositories.py +++ b/data_safe_haven/infrastructure/programs/sre/software_repositories.py @@ -1,4 +1,4 @@ -"""Pulumi component for SRE monitoring""" +"""Pulumi component for SRE software repositories""" from collections.abc import Mapping