From fd6dfec1f5330fb2b014dd79d92cf09f66e3ace0 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 11:57:32 +0000 Subject: [PATCH 1/8] Add diagnostic setting for NFSv3 containers --- .../composite/nfsv3_blob_container.py | 44 ++++++++++++++++++- .../infrastructure/programs/sre/data.py | 5 +++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index 98564918a0..f4275fed8e 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,9 +1,10 @@ from pulumi import ComponentResource, Input, ResourceOptions -from pulumi_azure_native import storage +from pulumi_azure_native import insights, storage -from data_safe_haven.infrastructure.components.dynamic.blob_container_acl import ( +from data_safe_haven.infrastructure.components import ( BlobContainerAcl, BlobContainerAclProps, + WrappedLogAnalyticsWorkspace, ) @@ -15,6 +16,7 @@ def __init__( acl_other: Input[str], apply_default_permissions: Input[bool], container_name: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account: Input[storage.StorageAccount], subscription_name: Input[str], @@ -24,6 +26,7 @@ def __init__( self.acl_other = acl_other self.apply_default_permissions = apply_default_permissions self.container_name = container_name + self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account = storage_account self.subscription_name = subscription_name @@ -52,6 +55,7 @@ def __init__( ResourceOptions(parent=props.storage_account), ), ) + BlobContainerAcl( f"{storage_container._name}_acl", BlobContainerAclProps( @@ -70,6 +74,42 @@ def __init__( ), ) + insights.DiagnosticSetting( + f"{storage_container._name}_diagnostic_settings", + name="firewall_diagnostic_settings", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_container.id, + workspace_id=props.log_analytics_workspace.id, + ) + self.name = storage_container.name self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 711b76139f..f478b56df8 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -33,6 +33,7 @@ NFSV3BlobContainerProps, SSLCertificate, SSLCertificateProps, + WrappedLogAnalyticsWorkspace, WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -51,6 +52,7 @@ def __init__( dns_record: Input[network.RecordSet], dns_server_admin_password: Input[pulumi_random.RandomPassword], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], storage_quota_gb_home: Input[int], @@ -69,6 +71,7 @@ def __init__( self.dns_record = dns_record self.password_dns_server_admin = dns_server_admin_password self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -492,6 +495,7 @@ def __init__( # 65533 ownership of the fileshare (preventing use inside the SRE) apply_default_permissions=False, container_name="egress", + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, @@ -507,6 +511,7 @@ def __init__( # files (eg. with Azure Storage Explorer) apply_default_permissions=True, container_name="ingress", + log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, From 6dc5a688bf0efcf5f1ec1c053c14f26d78b3441e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 14:27:24 +0000 Subject: [PATCH 2/8] Correct imports --- .../components/composite/nfsv3_blob_container.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index f4275fed8e..96d6e8a8b0 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,9 +1,11 @@ from pulumi import ComponentResource, Input, ResourceOptions from pulumi_azure_native import insights, storage -from data_safe_haven.infrastructure.components import ( +from data_safe_haven.infrastructure.components.dynamic import ( BlobContainerAcl, BlobContainerAclProps, +) +from data_safe_haven.infrastructure.components.wrapped import ( WrappedLogAnalyticsWorkspace, ) From 9bc497f23b05d9a7ebd665e8f3d73280fde07ed5 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 14:36:10 +0000 Subject: [PATCH 3/8] Pass log workspace to data component --- .../programs/declarative_sre.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 78467f201b..96df9380b2 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -183,6 +183,20 @@ def __call__(self) -> None: tags=self.tags, ) + # Deploy monitoring + monitoring = SREMonitoringComponent( + "sre_monitoring", + self.stack_name, + SREMonitoringProps( + dns_private_zones=dns.private_zones, + location=self.config.azure.location, + resource_group_name=resource_group.name, + subnet=networking.subnet_monitoring, + timezone=self.config.sre.timezone, + ), + tags=self.tags, + ) + # Deploy data storage data = SREDataComponent( "sre_data", @@ -196,6 +210,7 @@ def __call__(self) -> None: dns_record=networking.shm_ns_record, dns_server_admin_password=dns.password_admin, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, storage_quota_gb_home=self.config.sre.storage_quota_gb.home, @@ -209,20 +224,6 @@ def __call__(self) -> None: tags=self.tags, ) - # Deploy monitoring - monitoring = SREMonitoringComponent( - "sre_monitoring", - self.stack_name, - SREMonitoringProps( - dns_private_zones=dns.private_zones, - location=self.config.azure.location, - resource_group_name=resource_group.name, - subnet=networking.subnet_monitoring, - timezone=self.config.sre.timezone, - ), - tags=self.tags, - ) - # Deploy the apt proxy server apt_proxy_server = SREAptProxyServerComponent( "sre_apt_proxy_server", From 78754ae57421113b1c8f95d85926c64380e3dd69 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Tue, 26 Nov 2024 15:35:53 +0000 Subject: [PATCH 4/8] Move diagnostic setting to storage account --- .../composite/nfsv3_blob_container.py | 43 +------------------ .../infrastructure/programs/sre/data.py | 41 +++++++++++++++++- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py index 96d6e8a8b0..29550e9541 100644 --- a/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_blob_container.py @@ -1,13 +1,10 @@ from pulumi import ComponentResource, Input, ResourceOptions -from pulumi_azure_native import insights, storage +from pulumi_azure_native import storage from data_safe_haven.infrastructure.components.dynamic import ( BlobContainerAcl, BlobContainerAclProps, ) -from data_safe_haven.infrastructure.components.wrapped import ( - WrappedLogAnalyticsWorkspace, -) class NFSV3BlobContainerProps: @@ -18,7 +15,6 @@ def __init__( acl_other: Input[str], apply_default_permissions: Input[bool], container_name: Input[str], - log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group_name: Input[str], storage_account: Input[storage.StorageAccount], subscription_name: Input[str], @@ -28,7 +24,6 @@ def __init__( self.acl_other = acl_other self.apply_default_permissions = apply_default_permissions self.container_name = container_name - self.log_analytics_workspace = log_analytics_workspace self.resource_group_name = resource_group_name self.storage_account = storage_account self.subscription_name = subscription_name @@ -76,42 +71,6 @@ def __init__( ), ) - insights.DiagnosticSetting( - f"{storage_container._name}_diagnostic_settings", - name="firewall_diagnostic_settings", - log_analytics_destination_type="Dedicated", - logs=[ - { - "category_group": "allLogs", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - { - "category_group": "audit", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - ], - metrics=[ - { - "category": "Transaction", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - } - ], - resource_uri=storage_container.id, - workspace_id=props.log_analytics_workspace.id, - ) - self.name = storage_container.name self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index f478b56df8..fcca697810 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,6 +7,7 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, + insights, keyvault, managedidentity, network, @@ -495,7 +496,6 @@ def __init__( # 65533 ownership of the fileshare (preventing use inside the SRE) apply_default_permissions=False, container_name="egress", - log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, @@ -511,12 +511,49 @@ def __init__( # files (eg. with Azure Storage Explorer) apply_default_permissions=True, container_name="ingress", - log_analytics_workspace=props.log_analytics_workspace, resource_group_name=props.resource_group_name, storage_account=storage_account_data_private_sensitive, subscription_name=props.subscription_name, ), ) + # Add diagnostic setting for blobs + insights.DiagnosticSetting( + f"{storage_account_data_private_sensitive._name}_diagnostic_setting", + name=f"{storage_account_data_private_sensitive._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_private_sensitive.id.apply( + lambda resource_id: resource_id + "/blobServices" + ), + workspace_id=props.log_analytics_workspace.id, + ) # Set up a private endpoint for the sensitive data storage account storage_account_data_private_sensitive_endpoint = network.PrivateEndpoint( f"{storage_account_data_private_sensitive._name}_private_endpoint", From 168ca245117aa1ff906a1319aab4ad9c11339ccd Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 11:01:54 +0000 Subject: [PATCH 5/8] Correct blobServices URI --- data_safe_haven/infrastructure/programs/sre/data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index fcca697810..5f68a8f637 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -550,7 +550,9 @@ def __init__( } ], resource_uri=storage_account_data_private_sensitive.id.apply( - lambda resource_id: resource_id + "/blobServices" + # This is the URI of the blobServices resource which is automatically + # created. + lambda resource_id: resource_id + "/blobServices/default" ), workspace_id=props.log_analytics_workspace.id, ) From 7887e0987ea575a8c2940cdb1014a7de0bf19a68 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 11:43:12 +0000 Subject: [PATCH 6/8] Move NFSv3 accounts to a component resource Diagnostic settings are created as part of the component. --- .../infrastructure/components/__init__.py | 6 +- .../components/composite/__init__.py | 6 + .../composite/nfsv3_storage_account.py | 144 ++++++++++++++++++ .../components/wrapped/__init__.py | 2 - .../wrapped/nfsv3_storage_account.py | 79 ---------- .../programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/data.py | 70 +++------ .../programs/sre/desired_state.py | 28 ++-- 8 files changed, 191 insertions(+), 145 deletions(-) create mode 100644 data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py delete mode 100644 data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py diff --git a/data_safe_haven/infrastructure/components/__init__.py b/data_safe_haven/infrastructure/components/__init__.py index f4b93b9c3d..52043d1ad3 100644 --- a/data_safe_haven/infrastructure/components/__init__.py +++ b/data_safe_haven/infrastructure/components/__init__.py @@ -9,6 +9,8 @@ MicrosoftSQLDatabaseProps, NFSV3BlobContainerComponent, NFSV3BlobContainerProps, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, PostgresqlDatabaseComponent, PostgresqlDatabaseProps, VMComponent, @@ -23,7 +25,6 @@ ) from .wrapped import ( WrappedLogAnalyticsWorkspace, - WrappedNFSV3StorageAccount, ) __all__ = [ @@ -41,11 +42,12 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", + "NFSV3StorageAccountComponent", + "NFSV3StorageAccountProps", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "SSLCertificate", "SSLCertificateProps", "VMComponent", "WrappedLogAnalyticsWorkspace", - "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/composite/__init__.py b/data_safe_haven/infrastructure/components/composite/__init__.py index bc09bc18a8..8e561dd73a 100644 --- a/data_safe_haven/infrastructure/components/composite/__init__.py +++ b/data_safe_haven/infrastructure/components/composite/__init__.py @@ -9,6 +9,10 @@ MicrosoftSQLDatabaseProps, ) from .nfsv3_blob_container import NFSV3BlobContainerComponent, NFSV3BlobContainerProps +from .nfsv3_storage_account import ( + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, +) from .postgresql_database import PostgresqlDatabaseComponent, PostgresqlDatabaseProps from .virtual_machine import LinuxVMComponentProps, VMComponent @@ -23,6 +27,8 @@ "MicrosoftSQLDatabaseProps", "NFSV3BlobContainerComponent", "NFSV3BlobContainerProps", + "NFSV3StorageAccountComponent", + "NFSV3StorageAccountProps", "PostgresqlDatabaseComponent", "PostgresqlDatabaseProps", "VMComponent", diff --git a/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py b/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py new file mode 100644 index 0000000000..ca003bbd3d --- /dev/null +++ b/data_safe_haven/infrastructure/components/composite/nfsv3_storage_account.py @@ -0,0 +1,144 @@ +from collections.abc import Mapping, Sequence + +from pulumi import ComponentResource, Input, Output, ResourceOptions +from pulumi_azure_native import insights, storage + +from data_safe_haven.external import AzureIPv4Range +from data_safe_haven.infrastructure.components.wrapped import ( + WrappedLogAnalyticsWorkspace, +) +from data_safe_haven.types import AzureServiceTag + + +class NFSV3StorageAccountProps: + def __init__( + self, + account_name: Input[str], + allowed_ip_addresses: Input[Sequence[str]] | None, + allowed_service_tag: AzureServiceTag | None, + location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], + resource_group_name: Input[str], + subnet_id: Input[str], + ): + self.account_name = account_name + self.allowed_ip_addresses = allowed_ip_addresses + self.allowed_service_tag = allowed_service_tag + self.location = location + self.log_analytics_workspace = log_analytics_workspace + self.resource_group_name = resource_group_name + self.subnet_id = subnet_id + + +class NFSV3StorageAccountComponent(ComponentResource): + encryption_args = storage.EncryptionArgs( + key_source=storage.KeySource.MICROSOFT_STORAGE, + services=storage.EncryptionServicesArgs( + blob=storage.EncryptionServiceArgs( + enabled=True, key_type=storage.KeyType.ACCOUNT + ), + file=storage.EncryptionServiceArgs( + enabled=True, key_type=storage.KeyType.ACCOUNT + ), + ), + ) + + def __init__( + self, + name: str, + props: NFSV3StorageAccountProps, + opts: ResourceOptions | None = None, + tags: Input[Mapping[str, Input[str]]] | None = None, + ): + super().__init__("dsh:sre:NFSV3StorageAccountComponent", name, {}, opts) + child_opts = ResourceOptions.merge(opts, ResourceOptions(parent=self)) + child_tags = {"component": "data"} | (tags if tags else {}) + + if props.allowed_service_tag == AzureServiceTag.INTERNET: + default_action = storage.DefaultAction.ALLOW + ip_rules = [] + else: + default_action = storage.DefaultAction.DENY + ip_rules = Output.from_input(props.allowed_ip_addresses).apply( + lambda ip_ranges: [ + storage.IPRuleArgs( + action=storage.Action.ALLOW, + i_p_address_or_range=str(ip_address), + ) + for ip_range in sorted(ip_ranges) + for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips() + ] + ) + + # Deploy storage account + self.storage_account = storage.StorageAccount( + f"{self._name}", + account_name=props.account_name, + allow_blob_public_access=False, + enable_https_traffic_only=True, + enable_nfs_v3=True, + encryption=self.encryption_args, + is_hns_enabled=True, + kind=storage.Kind.BLOCK_BLOB_STORAGE, + location=props.location, + minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, + network_rule_set=storage.NetworkRuleSetArgs( + bypass=storage.Bypass.AZURE_SERVICES, + default_action=default_action, + ip_rules=ip_rules, + virtual_network_rules=[ + storage.VirtualNetworkRuleArgs( + virtual_network_resource_id=props.subnet_id, + ) + ], + ), + public_network_access=storage.PublicNetworkAccess.ENABLED, + resource_group_name=props.resource_group_name, + sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS), + opts=child_opts, + tags=child_tags, + ) + + # Add diagnostic setting for blobs + insights.DiagnosticSetting( + f"{self.storage_account._name}_diagnostic_setting", + name=f"{self.storage_account._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=self.storage_account.id.apply( + # This is the URI of the blobServices resource which is automatically + # created. + lambda resource_id: resource_id + + "/blobServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) + + self.register_outputs({}) diff --git a/data_safe_haven/infrastructure/components/wrapped/__init__.py b/data_safe_haven/infrastructure/components/wrapped/__init__.py index b449f46859..fc5f8c8f61 100644 --- a/data_safe_haven/infrastructure/components/wrapped/__init__.py +++ b/data_safe_haven/infrastructure/components/wrapped/__init__.py @@ -1,7 +1,5 @@ from .log_analytics_workspace import WrappedLogAnalyticsWorkspace -from .nfsv3_storage_account import WrappedNFSV3StorageAccount __all__ = [ "WrappedLogAnalyticsWorkspace", - "WrappedNFSV3StorageAccount", ] diff --git a/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py b/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py deleted file mode 100644 index e259de4806..0000000000 --- a/data_safe_haven/infrastructure/components/wrapped/nfsv3_storage_account.py +++ /dev/null @@ -1,79 +0,0 @@ -from collections.abc import Mapping, Sequence - -from pulumi import Input, Output, ResourceOptions -from pulumi_azure_native import storage - -from data_safe_haven.external import AzureIPv4Range -from data_safe_haven.types import AzureServiceTag - - -class WrappedNFSV3StorageAccount(storage.StorageAccount): - encryption_args = storage.EncryptionArgs( - key_source=storage.KeySource.MICROSOFT_STORAGE, - services=storage.EncryptionServicesArgs( - blob=storage.EncryptionServiceArgs( - enabled=True, key_type=storage.KeyType.ACCOUNT - ), - file=storage.EncryptionServiceArgs( - enabled=True, key_type=storage.KeyType.ACCOUNT - ), - ), - ) - - def __init__( - self, - resource_name: str, - *, - account_name: Input[str], - allowed_ip_addresses: Input[Sequence[str]] | None, - allowed_service_tag: AzureServiceTag | None, - location: Input[str], - resource_group_name: Input[str], - subnet_id: Input[str], - opts: ResourceOptions, - tags: Input[Mapping[str, Input[str]]], - ): - if allowed_service_tag == AzureServiceTag.INTERNET: - default_action = storage.DefaultAction.ALLOW - ip_rules = [] - else: - default_action = storage.DefaultAction.DENY - ip_rules = Output.from_input(allowed_ip_addresses).apply( - lambda ip_ranges: [ - storage.IPRuleArgs( - action=storage.Action.ALLOW, - i_p_address_or_range=str(ip_address), - ) - for ip_range in sorted(ip_ranges) - for ip_address in AzureIPv4Range.from_cidr(ip_range).all_ips() - ] - ) - - self.resource_group_name_ = Output.from_input(resource_group_name) - super().__init__( - resource_name, - account_name=account_name, - allow_blob_public_access=False, - enable_https_traffic_only=True, - enable_nfs_v3=True, - encryption=self.encryption_args, - is_hns_enabled=True, - kind=storage.Kind.BLOCK_BLOB_STORAGE, - location=location, - minimum_tls_version=storage.MinimumTlsVersion.TLS1_2, - network_rule_set=storage.NetworkRuleSetArgs( - bypass=storage.Bypass.AZURE_SERVICES, - default_action=default_action, - ip_rules=ip_rules, - virtual_network_rules=[ - storage.VirtualNetworkRuleArgs( - virtual_network_resource_id=subnet_id, - ) - ], - ), - public_network_access=storage.PublicNetworkAccess.ENABLED, - resource_group_name=resource_group_name, - sku=storage.SkuArgs(name=storage.SkuName.PREMIUM_ZRS), - opts=opts, - tags=tags, - ) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index 96df9380b2..da796adfcc 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -377,6 +377,7 @@ def __call__(self) -> None: ldap_user_filter=ldap_user_filter, ldap_user_search_base=ldap_user_search_base, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, software_repository_hostname=user_services.software_repositories.hostname, subnet_desired_state=networking.subnet_desired_state, diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 5f68a8f637..645ea8fd87 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,7 +7,6 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, - insights, keyvault, managedidentity, network, @@ -32,10 +31,11 @@ from data_safe_haven.infrastructure.components import ( NFSV3BlobContainerComponent, NFSV3BlobContainerProps, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, SSLCertificate, SSLCertificateProps, WrappedLogAnalyticsWorkspace, - WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -471,20 +471,26 @@ def __init__( # Deploy sensitive data blob storage account # - This holds the /mnt/input and /mnt/output containers that are mounted by workspaces # - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer - storage_account_data_private_sensitive = WrappedNFSV3StorageAccount( + component_data_private_sensitive = NFSV3StorageAccountComponent( f"{self._name}_storage_account_data_private_sensitive", - # Storage account names have a maximum of 24 characters - account_name=alphanumeric( - f"{''.join(truncate_tokens(stack_name.split('-'), 11))}sensitivedata{sha256hash(self._name)}" - )[:24], - allowed_ip_addresses=data_private_sensitive_ip_addresses, - allowed_service_tag=data_private_sensitive_service_tag, - location=props.location, - subnet_id=props.subnet_data_private_id, - resource_group_name=props.resource_group_name, + NFSV3StorageAccountProps( + # Storage account names have a maximum of 24 characters + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 11))}sensitivedata{sha256hash(self._name)}" + )[:24], + allowed_ip_addresses=data_private_sensitive_ip_addresses, + allowed_service_tag=data_private_sensitive_service_tag, + location=props.location, + log_analytics_workspace=props.log_analytics_workspace, + subnet_id=props.subnet_data_private_id, + resource_group_name=props.resource_group_name, + ), opts=child_opts, tags=child_tags, ) + storage_account_data_private_sensitive = ( + component_data_private_sensitive.storage_account + ) # Deploy storage containers NFSV3BlobContainerComponent( f"{self._name}_blob_egress", @@ -516,46 +522,6 @@ def __init__( subscription_name=props.subscription_name, ), ) - # Add diagnostic setting for blobs - insights.DiagnosticSetting( - f"{storage_account_data_private_sensitive._name}_diagnostic_setting", - name=f"{storage_account_data_private_sensitive._name}_diagnostic_setting", - log_analytics_destination_type="Dedicated", - logs=[ - { - "category_group": "allLogs", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - { - "category_group": "audit", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - }, - ], - metrics=[ - { - "category": "Transaction", - "enabled": True, - "retention_policy": { - "days": 0, - "enabled": False, - }, - } - ], - resource_uri=storage_account_data_private_sensitive.id.apply( - # This is the URI of the blobServices resource which is automatically - # created. - lambda resource_id: resource_id + "/blobServices/default" - ), - workspace_id=props.log_analytics_workspace.id, - ) # Set up a private endpoint for the sensitive data storage account storage_account_data_private_sensitive_endpoint = network.PrivateEndpoint( f"{storage_account_data_private_sensitive._name}_private_endpoint", diff --git a/data_safe_haven/infrastructure/programs/sre/desired_state.py b/data_safe_haven/infrastructure/programs/sre/desired_state.py index c4392f5210..20f4e357f1 100644 --- a/data_safe_haven/infrastructure/programs/sre/desired_state.py +++ b/data_safe_haven/infrastructure/programs/sre/desired_state.py @@ -31,7 +31,9 @@ from data_safe_haven.infrastructure.components import ( NFSV3BlobContainerComponent, NFSV3BlobContainerProps, - WrappedNFSV3StorageAccount, + NFSV3StorageAccountComponent, + NFSV3StorageAccountProps, + WrappedLogAnalyticsWorkspace, ) from data_safe_haven.resources import resources_path from data_safe_haven.types import AzureDnsZoneNames @@ -55,6 +57,7 @@ def __init__( ldap_user_filter: Input[str], ldap_user_search_base: Input[str], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], software_repository_hostname: Input[str], subscription_name: Input[str], @@ -73,6 +76,7 @@ def __init__( self.ldap_user_filter = ldap_user_filter self.ldap_user_search_base = ldap_user_search_base self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -102,19 +106,23 @@ def __init__( # Deploy desired state storage account # - This holds the /var/local/ansible container that is mounted by workspaces # - Azure blobs have worse NFS support but can be accessed with Azure Storage Explorer - storage_account = WrappedNFSV3StorageAccount( + storage_component = NFSV3StorageAccountComponent( f"{self._name}_storage_account", - account_name=alphanumeric( - f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}" - )[:24], - allowed_ip_addresses=props.admin_ip_addresses, - allowed_service_tag=None, - location=props.location, - resource_group_name=props.resource_group_name, - subnet_id=props.subnet_desired_state_id, + NFSV3StorageAccountProps( + account_name=alphanumeric( + f"{''.join(truncate_tokens(stack_name.split('-'), 11))}desiredstate{sha256hash(self._name)}" + )[:24], + allowed_ip_addresses=props.admin_ip_addresses, + allowed_service_tag=None, + location=props.location, + log_analytics_workspace=props.log_analytics_workspace, + resource_group_name=props.resource_group_name, + subnet_id=props.subnet_desired_state_id, + ), opts=child_opts, tags=child_tags, ) + storage_account = storage_component.storage_account # Deploy desired state share container_desired_state = NFSV3BlobContainerComponent( f"{self._name}_blob_desired_state", From 696b192ad1c65dcb9ff4fea922d2ba5628847702 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 14:01:57 +0000 Subject: [PATCH 7/8] Add blob log documentation --- docs/source/management/logs.md | 38 +++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 80852848bd..1b6876794c 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -11,6 +11,38 @@ Select which Log Analytics Workspace you want to view by clicking on the workspa The logs can be filtered using [Kusto Query Language (KQL)](https://learn.microsoft.com/en-us/azure/azure-monitor/logs/log-query-overview). +## Storage logs + +Depending on how different parts of Data Safe Haven storage are provisioned, logs may differ. + +### Sensitive data logs + +The sensitive data containers are the [ingress and egress containers](./data.md). +Logs from these containers are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageBlobLogs` +: Events occurring on the blob containers. +: For example data being uploaded, extracted or read. + +`AzureMetrics` +: Various metrics on blob container utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. + +### Desired state data logs + +The desired state container holds the data necessary to configure virtual machines in an SRE. +Logs from the desired state container are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageBlobLogs` +: Events occurring on the blob containers. +: For example data being uploaded, extracted or read. + +`AzureMetrics` +: Various metrics on blob container utilisation and performance. +: This table is not reserved for the firewall and other resources may log to it. + ## Container logs Some of the Data Safe Haven infrastructure is provisioned as containers. @@ -21,7 +53,7 @@ These include, - Gitea and Hedgedoc Logs from all containers are ingested into the [SRE's log analytics workspace](#log-workspace). -There are two logs +There are two tables, `ContainerEvents_CL` : Event logs for the container instance resources such as starting, stopping, crashes and pulling images. @@ -34,7 +66,7 @@ There are two logs Logs from all user workspaces are ingested into the [SRE's log analytics workspace](#log-workspace) using the [Azure Monitor Agent](https://learn.microsoft.com/en-us/azure/azure-monitor/agents/azure-monitor-agent-overview). -There are three logs +There are three tables, `Perf` : Usage statistics for individual workspaces, such as percent memory used and percent disk space used. @@ -53,7 +85,7 @@ The firewall plays a critical role in the security of a Data Safe Haven. It filters all outbound traffic through a set of FQDN rules so that each component may only reach necessary and allowed domains. Logs from the firewall are ingested into the [SREs log workspace](#log-workspace). -There are multiple tables, +There are three tables, `AZFWApplicationRule` : Logs from the firewalls FDQN filters. From 37fdeb7e04d82eb42c4fd08e6b5a83f522692e4e Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:56:55 +0000 Subject: [PATCH 8/8] Correct descriptions --- docs/source/management/logs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 1b6876794c..d7607cf98a 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -27,7 +27,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the sensitive data containers and other resources may log to it. ### Desired state data logs @@ -41,7 +41,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the desired state data container and other resources may log to it. ## Container logs