From 04d838faa46e78fd31cc20a4327e22b5041d1807 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 14:55:44 +0000 Subject: [PATCH 1/7] Add diagnostic setting for user data shares --- .../programs/declarative_sre.py | 1 + .../infrastructure/programs/sre/data.py | 45 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py index f69fc9cd45..8cca2dff85 100644 --- a/data_safe_haven/infrastructure/programs/declarative_sre.py +++ b/data_safe_haven/infrastructure/programs/declarative_sre.py @@ -211,6 +211,7 @@ def __call__(self) -> None: dns_record=networking.shm_ns_record, dns_server_admin_password=dns.password_admin, location=self.config.azure.location, + log_analytics_workspace=monitoring.log_analytics, resource_group=resource_group, sre_fqdn=networking.sre_fqdn, storage_quota_gb_home=self.config.sre.storage_quota_gb.home, diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 711b76139f..8e1a278770 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -7,6 +7,7 @@ from pulumi import ComponentResource, Input, Output, ResourceOptions from pulumi_azure_native import ( authorization, + insights, keyvault, managedidentity, network, @@ -33,6 +34,7 @@ NFSV3BlobContainerProps, SSLCertificate, SSLCertificateProps, + WrappedLogAnalyticsWorkspace, WrappedNFSV3StorageAccount, ) from data_safe_haven.types import AzureDnsZoneNames, AzureServiceTag @@ -51,6 +53,7 @@ def __init__( dns_record: Input[network.RecordSet], dns_server_admin_password: Input[pulumi_random.RandomPassword], location: Input[str], + log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace], resource_group: Input[resources.ResourceGroup], sre_fqdn: Input[str], storage_quota_gb_home: Input[int], @@ -69,6 +72,7 @@ def __init__( self.dns_record = dns_record self.password_dns_server_admin = dns_server_admin_password self.location = location + self.log_analytics_workspace = log_analytics_workspace self.resource_group_id = Output.from_input(resource_group).apply(get_id_from_rg) self.resource_group_name = Output.from_input(resource_group).apply( get_name_from_rg @@ -615,6 +619,47 @@ def __init__( opts=child_opts, tags=child_tags, ) + # Add diagnostic setting for files + insights.DiagnosticSetting( + f"{storage_account_data_private_user._name}_diagnostic_setting", + name=f"{storage_account_data_private_user._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_private_user.id.apply( + # This is the URI of the fileServices resource which is automatically + # created. + lambda resource_id: resource_id + + "/fileServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) storage.FileShare( f"{storage_account_data_private_user._name}_files_home", access_tier=storage.ShareAccessTier.PREMIUM, From b90a1ff9342f12f665a071fe0b2e04c792b9978d Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Thu, 28 Nov 2024 15:03:12 +0000 Subject: [PATCH 2/7] Add diagnostic setting for config file shares --- .../infrastructure/programs/sre/data.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index 8e1a278770..0dabd47f71 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -425,6 +425,47 @@ def __init__( resource_group_name=kwargs["resource_group_name"], ) ) + # Add diagnostic setting for files + insights.DiagnosticSetting( + f"{storage_account_data_configuration._name}_diagnostic_setting", + name=f"{storage_account_data_configuration._name}_diagnostic_setting", + log_analytics_destination_type="Dedicated", + logs=[ + { + "category_group": "allLogs", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + { + "category_group": "audit", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + }, + ], + metrics=[ + { + "category": "Transaction", + "enabled": True, + "retention_policy": { + "days": 0, + "enabled": False, + }, + } + ], + resource_uri=storage_account_data_configuration.id.apply( + # This is the URI of the fileService resource which is automatically + # created. + lambda resource_id: resource_id + + "/fileServices/default" + ), + workspace_id=props.log_analytics_workspace.id, + ) # Set up a private endpoint for the configuration data storage account storage_account_data_configuration_private_endpoint = network.PrivateEndpoint( f"{storage_account_data_configuration._name}_private_endpoint", From 2e56565d77c243b8113a397f0d06e9a5463ebdcc Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:55:55 +0000 Subject: [PATCH 3/7] Add documentation for file share logs --- docs/source/management/logs.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 1b6876794c..408ec9b631 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -43,6 +43,35 @@ There are two tables, : Various metrics on blob container utilisation and performance. : This table is not reserved for the firewall and other resources may log to it. +### User data logs + +The user data file share holds the {ref}`role_researcher`s' [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. +Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageFileLogs` +: NFS events occurring on the file share. +: For example data being written or directories being accessed + +`AzureMetrics` +: Various metrics on file share utilisation and performance. +: This table is not reserved for the user data share and other resources may log to it. + +### Configuration data logs + +There are multiple configuration data file shares. +Each contains the configuration and state data for the Data Safe Haven [services deployed as containers](#container-logs). +Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). +There are two tables, + +`StorageFileLogs` +: SMB events occurring on the file share. +: For example data being written or directories being accessed + +`AzureMetrics` +: Various metrics on file share utilisation and performance. +: This table is not reserved for the configuration data shares and other resources may log to it. + ## Container logs Some of the Data Safe Haven infrastructure is provisioned as containers. From 799f96b7944145e40d4c7889bd30f5374c6cf4a9 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 09:56:55 +0000 Subject: [PATCH 4/7] Correct descriptions --- docs/source/management/logs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 408ec9b631..63314654fc 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -27,7 +27,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the sensitive data containers and other resources may log to it. ### Desired state data logs @@ -41,7 +41,7 @@ There are two tables, `AzureMetrics` : Various metrics on blob container utilisation and performance. -: This table is not reserved for the firewall and other resources may log to it. +: This table is not reserved for the desired state data container and other resources may log to it. ### User data logs From d5802fb893a0d56ab2256229704ab5ce316c1917 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:03:51 +0000 Subject: [PATCH 5/7] Improve reference rendering --- docs/source/management/logs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/management/logs.md b/docs/source/management/logs.md index 63314654fc..10b9bfb0e5 100644 --- a/docs/source/management/logs.md +++ b/docs/source/management/logs.md @@ -45,7 +45,7 @@ There are two tables, ### User data logs -The user data file share holds the {ref}`role_researcher`s' [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. +The user data file share holds the {ref}`researchers'` [home directories](https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s08.html), where they will store their personal data and configuration. Logs from the share are ingested into the [SRE's log analytics workspace](#log-workspace). There are two tables, From da382e0e600991e4e19728760d5b4e048699a3d1 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:15:59 +0000 Subject: [PATCH 6/7] Use Output concat method Co-authored-by: James Robinson --- .../infrastructure/programs/sre/data.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index e348f25976..a789d61ad0 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -459,12 +459,8 @@ def __init__( }, } ], - resource_uri=storage_account_data_configuration.id.apply( - # This is the URI of the fileService resource which is automatically - # created. - lambda resource_id: resource_id - + "/fileServices/default" - ), + # This is the URI of the automatically created fileService resource + resource_uri=Output.concat(storage_account_data_configuration.id, "/fileServices/default"), workspace_id=props.log_analytics_workspace.id, ) # Set up a private endpoint for the configuration data storage account @@ -700,12 +696,8 @@ def __init__( }, } ], - resource_uri=storage_account_data_private_user.id.apply( - # This is the URI of the fileServices resource which is automatically - # created. - lambda resource_id: resource_id - + "/fileServices/default" - ), + # This is the URI of the automatically created fileService resource + resource_uri=Output.concat(storage_account_data_private_user.id, "/fileServices/default"), workspace_id=props.log_analytics_workspace.id, ) storage.FileShare( From d73704a0742402f638430e6c321e2872f1ff7bb4 Mon Sep 17 00:00:00 2001 From: Jim Madge Date: Fri, 29 Nov 2024 10:18:14 +0000 Subject: [PATCH 7/7] Fix linting --- data_safe_haven/infrastructure/programs/sre/data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/data_safe_haven/infrastructure/programs/sre/data.py b/data_safe_haven/infrastructure/programs/sre/data.py index a789d61ad0..825861c122 100644 --- a/data_safe_haven/infrastructure/programs/sre/data.py +++ b/data_safe_haven/infrastructure/programs/sre/data.py @@ -460,7 +460,9 @@ def __init__( } ], # This is the URI of the automatically created fileService resource - resource_uri=Output.concat(storage_account_data_configuration.id, "/fileServices/default"), + resource_uri=Output.concat( + storage_account_data_configuration.id, "/fileServices/default" + ), workspace_id=props.log_analytics_workspace.id, ) # Set up a private endpoint for the configuration data storage account @@ -697,7 +699,9 @@ def __init__( } ], # This is the URI of the automatically created fileService resource - resource_uri=Output.concat(storage_account_data_private_user.id, "/fileServices/default"), + resource_uri=Output.concat( + storage_account_data_private_user.id, "/fileServices/default" + ), workspace_id=props.log_analytics_workspace.id, ) storage.FileShare(