Merge pull request #2295 from alan-turing-institute/container_logs

Add logging for container instances
alan-turing-institute · Nov 21, 2024 · 1a9b6bf · 1a9b6bf
2 parents d1f9d98 + 90f914c
commit 1a9b6bf
Show file tree

Hide file tree

Showing 17 changed files with 373 additions and 263 deletions.
diff --git a/.mdlstyle.rb b/.mdlstyle.rb
@@ -6,7 +6,7 @@
 exclude_rule 'MD013'
 exclude_rule 'MD024'
 rule 'MD026', :punctuation => ".,;"
-rule 'MD029', :style => :ordered
+rule 'MD029', :style => :one
 exclude_rule 'MD033'
 exclude_rule 'MD034'
 exclude_rule 'MD041' # this conflicts with MyST target anchors
diff --git a/data_safe_haven/infrastructure/programs/declarative_sre.py b/data_safe_haven/infrastructure/programs/declarative_sre.py
@@ -209,6 +209,20 @@ def __call__(self) -> None:
             tags=self.tags,
         )
 
+        # Deploy monitoring
+        monitoring = SREMonitoringComponent(
+            "sre_monitoring",
+            self.stack_name,
+            SREMonitoringProps(
+                dns_private_zones=dns.private_zones,
+                location=self.config.azure.location,
+                resource_group_name=resource_group.name,
+                subnet=networking.subnet_monitoring,
+                timezone=self.config.sre.timezone,
+            ),
+            tags=self.tags,
+        )
+
         # Deploy the apt proxy server
         apt_proxy_server = SREAptProxyServerComponent(
             "sre_apt_proxy_server",
@@ -217,6 +231,7 @@ def __call__(self) -> None:
                 containers_subnet=networking.subnet_apt_proxy_server,
                 dns_server_ip=dns.ip_address,
                 location=self.config.azure.location,
+                log_analytics_workspace=monitoring.log_analytics,
                 resource_group_name=resource_group.name,
                 sre_fqdn=networking.sre_fqdn,
                 storage_account_key=data.storage_account_data_configuration_key,
@@ -233,6 +248,7 @@ def __call__(self) -> None:
                 dns_server_ip=dns.ip_address,
                 dockerhub_credentials=dockerhub_credentials,
                 location=self.config.azure.location,
+                log_analytics_workspace=monitoring.log_analytics,
                 resource_group_name=resource_group.name,
                 sre_fqdn=networking.sre_fqdn,
                 storage_account_key=data.storage_account_data_configuration_key,
@@ -253,6 +269,7 @@ def __call__(self) -> None:
                 entra_application_secret=entra.identity_application_secret,
                 entra_tenant_id=shm_entra_tenant_id,
                 location=self.config.azure.location,
+                log_analytics_workspace=monitoring.log_analytics,
                 resource_group_name=resource_group.name,
                 shm_fqdn=shm_fqdn,
                 sre_fqdn=networking.sre_fqdn,
@@ -299,6 +316,7 @@ def __call__(self) -> None:
                 ldap_user_filter=ldap_user_filter,
                 ldap_user_search_base=ldap_user_search_base,
                 location=self.config.azure.location,
+                log_analytics_workspace=monitoring.log_analytics,
                 resource_group_name=resource_group.name,
                 storage_account_key=data.storage_account_data_configuration_key,
                 storage_account_name=data.storage_account_data_configuration_name,
@@ -325,6 +343,7 @@ def __call__(self) -> None:
                 ldap_username_attribute=ldap_username_attribute,
                 ldap_user_search_base=ldap_user_search_base,
                 location=self.config.azure.location,
+                log_analytics_workspace=monitoring.log_analytics,
                 nexus_admin_password=data.password_nexus_admin,
                 resource_group_name=resource_group.name,
                 software_packages=self.config.sre.software_packages,
@@ -339,20 +358,6 @@ def __call__(self) -> None:
             tags=self.tags,
         )
 
-        # Deploy monitoring
-        monitoring = SREMonitoringComponent(
-            "sre_monitoring",
-            self.stack_name,
-            SREMonitoringProps(
-                dns_private_zones=dns.private_zones,
-                location=self.config.azure.location,
-                resource_group_name=resource_group.name,
-                subnet=networking.subnet_monitoring,
-                timezone=self.config.sre.timezone,
-            ),
-            tags=self.tags,
-        )
-
         # Deploy desired state
         desired_state = SREDesiredStateComponent(
             "sre_desired_state",

diff --git a/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py b/data_safe_haven/infrastructure/programs/sre/apt_proxy_server.py
@@ -12,6 +12,7 @@
     FileShareFileProps,
     LocalDnsRecordComponent,
     LocalDnsRecordProps,
+    WrappedLogAnalyticsWorkspace,
 )
 from data_safe_haven.types import PermittedDomains
 
@@ -24,6 +25,7 @@ def __init__(
         containers_subnet: Input[str],
         dns_server_ip: Input[str],
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         resource_group_name: Input[str],
         sre_fqdn: Input[str],
         storage_account_key: Input[str],
@@ -34,6 +36,7 @@ def __init__(
         )
         self.dns_server_ip = dns_server_ip
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.resource_group_name = resource_group_name
         self.sre_fqdn = sre_fqdn
         self.storage_account_key = storage_account_key
@@ -119,6 +122,12 @@ def __init__(
                     ],
                 ),
             ],
+            diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                log_analytics=containerinstance.LogAnalyticsArgs(
+                    workspace_id=props.log_analytics_workspace.workspace_id,
+                    workspace_key=props.log_analytics_workspace.workspace_key,
+                ),
+            ),
             dns_config=containerinstance.DnsConfigurationArgs(
                 name_servers=[props.dns_server_ip],
             ),

diff --git a/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py b/data_safe_haven/infrastructure/programs/sre/clamav_mirror.py
@@ -11,6 +11,7 @@
 from data_safe_haven.infrastructure.components import (
     LocalDnsRecordComponent,
     LocalDnsRecordProps,
+    WrappedLogAnalyticsWorkspace,
 )
 
 
@@ -22,6 +23,7 @@ def __init__(
         dns_server_ip: Input[str],
         dockerhub_credentials: DockerHubCredentials,
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         resource_group_name: Input[str],
         sre_fqdn: Input[str],
         storage_account_key: Input[str],
@@ -31,6 +33,7 @@ def __init__(
         self.dns_server_ip = dns_server_ip
         self.dockerhub_credentials = dockerhub_credentials
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.resource_group_name = resource_group_name
         self.sre_fqdn = sre_fqdn
         self.storage_account_key = storage_account_key
@@ -95,6 +98,12 @@ def __init__(
                     ],
                 ),
             ],
+            diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                log_analytics=containerinstance.LogAnalyticsArgs(
+                    workspace_id=props.log_analytics_workspace.workspace_id,
+                    workspace_key=props.log_analytics_workspace.workspace_key,
+                ),
+            ),
             dns_config=containerinstance.DnsConfigurationArgs(
                 name_servers=[props.dns_server_ip],
             ),

diff --git a/data_safe_haven/infrastructure/programs/sre/gitea_server.py b/data_safe_haven/infrastructure/programs/sre/gitea_server.py
@@ -14,6 +14,7 @@
     LocalDnsRecordProps,
     PostgresqlDatabaseComponent,
     PostgresqlDatabaseProps,
+    WrappedLogAnalyticsWorkspace,
 )
 from data_safe_haven.resources import resources_path
 from data_safe_haven.utility import FileReader
@@ -35,6 +36,7 @@ def __init__(
         ldap_user_filter: Input[str],
         ldap_user_search_base: Input[str],
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         resource_group_name: Input[str],
         sre_fqdn: Input[str],
         storage_account_key: Input[str],
@@ -55,6 +57,7 @@ def __init__(
         self.ldap_user_filter = ldap_user_filter
         self.ldap_user_search_base = ldap_user_search_base
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.resource_group_name = resource_group_name
         self.sre_fqdn = sre_fqdn
         self.storage_account_key = storage_account_key
@@ -272,6 +275,12 @@ def __init__(
                     ],
                 ),
             ],
+            diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                log_analytics=containerinstance.LogAnalyticsArgs(
+                    workspace_id=props.log_analytics_workspace.workspace_id,
+                    workspace_key=props.log_analytics_workspace.workspace_key,
+                ),
+            ),
             dns_config=containerinstance.DnsConfigurationArgs(
                 name_servers=[props.dns_server_ip],
             ),

diff --git a/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py b/data_safe_haven/infrastructure/programs/sre/hedgedoc_server.py
@@ -15,6 +15,7 @@
     LocalDnsRecordProps,
     PostgresqlDatabaseComponent,
     PostgresqlDatabaseProps,
+    WrappedLogAnalyticsWorkspace,
 )
 from data_safe_haven.resources import resources_path
 from data_safe_haven.types import Ports
@@ -37,6 +38,7 @@ def __init__(
         ldap_user_search_base: Input[str],
         ldap_username_attribute: Input[str],
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         resource_group_name: Input[str],
         sre_fqdn: Input[str],
         storage_account_key: Input[str],
@@ -58,6 +60,7 @@ def __init__(
         self.ldap_user_search_base = ldap_user_search_base
         self.ldap_username_attribute = ldap_username_attribute
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.resource_group_name = resource_group_name
         self.sre_fqdn = sre_fqdn
         self.storage_account_key = storage_account_key
@@ -253,6 +256,12 @@ def __init__(
                     ],
                 ),
             ],
+            diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                log_analytics=containerinstance.LogAnalyticsArgs(
+                    workspace_id=props.log_analytics_workspace.workspace_id,
+                    workspace_key=props.log_analytics_workspace.workspace_key,
+                ),
+            ),
             dns_config=containerinstance.DnsConfigurationArgs(
                 name_servers=[props.dns_server_ip],
             ),

diff --git a/data_safe_haven/infrastructure/programs/sre/identity.py b/data_safe_haven/infrastructure/programs/sre/identity.py
@@ -13,6 +13,7 @@
 from data_safe_haven.infrastructure.components import (
     LocalDnsRecordComponent,
     LocalDnsRecordProps,
+    WrappedLogAnalyticsWorkspace,
 )
 
 
@@ -27,6 +28,7 @@ def __init__(
         entra_application_secret: Input[str],
         entra_tenant_id: Input[str],
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         resource_group_name: Input[str],
         shm_fqdn: Input[str],
         sre_fqdn: Input[str],
@@ -40,6 +42,7 @@ def __init__(
         self.entra_application_secret = entra_application_secret
         self.entra_tenant_id = entra_tenant_id
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.resource_group_name = resource_group_name
         self.shm_fqdn = shm_fqdn
         self.sre_fqdn = sre_fqdn
@@ -163,6 +166,12 @@ def __init__(
                     ],
                 ),
             ],
+            diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                log_analytics=containerinstance.LogAnalyticsArgs(
+                    workspace_id=props.log_analytics_workspace.workspace_id,
+                    workspace_key=props.log_analytics_workspace.workspace_key,
+                ),
+            ),
             dns_config=containerinstance.DnsConfigurationArgs(
                 name_servers=[props.dns_server_ip],
             ),

diff --git a/data_safe_haven/infrastructure/programs/sre/remote_desktop.py b/data_safe_haven/infrastructure/programs/sre/remote_desktop.py
@@ -15,6 +15,7 @@
     FileShareFileProps,
     PostgresqlDatabaseComponent,
     PostgresqlDatabaseProps,
+    WrappedLogAnalyticsWorkspace,
 )
 from data_safe_haven.resources import resources_path
 from data_safe_haven.utility import FileReader
@@ -40,6 +41,7 @@ def __init__(
         ldap_user_filter: Input[str],
         ldap_user_search_base: Input[str],
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         resource_group_name: Input[str],
         storage_account_key: Input[str],
         storage_account_name: Input[str],
@@ -65,6 +67,7 @@ def __init__(
         self.ldap_user_filter = ldap_user_filter
         self.ldap_user_search_base = ldap_user_search_base
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.resource_group_name = resource_group_name
         self.storage_account_key = storage_account_key
         self.storage_account_name = storage_account_name
@@ -348,6 +351,12 @@ def __init__(
                     ),
                 ),
             ],
+            diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                log_analytics=containerinstance.LogAnalyticsArgs(
+                    workspace_id=props.log_analytics_workspace.workspace_id,
+                    workspace_key=props.log_analytics_workspace.workspace_key,
+                ),
+            ),
             dns_config=containerinstance.DnsConfigurationArgs(
                 name_servers=[props.dns_server_ip],
             ),

diff --git a/data_safe_haven/infrastructure/programs/sre/software_repositories.py b/data_safe_haven/infrastructure/programs/sre/software_repositories.py
@@ -14,6 +14,7 @@
     FileShareFileProps,
     LocalDnsRecordComponent,
     LocalDnsRecordProps,
+    WrappedLogAnalyticsWorkspace,
 )
 from data_safe_haven.resources import resources_path
 from data_safe_haven.types import Ports, SoftwarePackageCategory
@@ -28,6 +29,7 @@ def __init__(
         dns_server_ip: Input[str],
         dockerhub_credentials: DockerHubCredentials,
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         nexus_admin_password: Input[str],
         resource_group_name: Input[str],
         software_packages: SoftwarePackageCategory,
@@ -39,6 +41,7 @@ def __init__(
         self.dns_server_ip = dns_server_ip
         self.dockerhub_credentials = dockerhub_credentials
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.nexus_admin_password = Output.secret(nexus_admin_password)
         self.nexus_packages: str | None = {
             SoftwarePackageCategory.ANY: "all",
@@ -250,6 +253,12 @@ def __init__(
                         ],
                     ),
                 ],
+                diagnostics=containerinstance.ContainerGroupDiagnosticsArgs(
+                    log_analytics=containerinstance.LogAnalyticsArgs(
+                        workspace_id=props.log_analytics_workspace.workspace_id,
+                        workspace_key=props.log_analytics_workspace.workspace_key,
+                    ),
+                ),
                 dns_config=containerinstance.DnsConfigurationArgs(
                     name_servers=[props.dns_server_ip],
                 ),

diff --git a/data_safe_haven/infrastructure/programs/sre/user_services.py b/data_safe_haven/infrastructure/programs/sre/user_services.py
@@ -7,6 +7,7 @@
     DockerHubCredentials,
     get_id_from_subnet,
 )
+from data_safe_haven.infrastructure.components import WrappedLogAnalyticsWorkspace
 from data_safe_haven.types import DatabaseSystem, SoftwarePackageCategory
 
 from .database_servers import SREDatabaseServerComponent, SREDatabaseServerProps
@@ -35,6 +36,7 @@ def __init__(
         ldap_user_filter: Input[str],
         ldap_user_search_base: Input[str],
         location: Input[str],
+        log_analytics_workspace: Input[WrappedLogAnalyticsWorkspace],
         nexus_admin_password: Input[str],
         resource_group_name: Input[str],
         software_packages: SoftwarePackageCategory,
@@ -58,6 +60,7 @@ def __init__(
         self.ldap_user_filter = ldap_user_filter
         self.ldap_user_search_base = ldap_user_search_base
         self.location = location
+        self.log_analytics_workspace = log_analytics_workspace
         self.nexus_admin_password = Output.secret(nexus_admin_password)
         self.resource_group_name = resource_group_name
         self.software_packages = software_packages
@@ -109,6 +112,7 @@ def __init__(
                 ldap_user_filter=props.ldap_user_filter,
                 ldap_user_search_base=props.ldap_user_search_base,
                 location=props.location,
+                log_analytics_workspace=props.log_analytics_workspace,
                 resource_group_name=props.resource_group_name,
                 sre_fqdn=props.sre_fqdn,
                 storage_account_key=props.storage_account_key,
@@ -134,6 +138,7 @@ def __init__(
                 ldap_user_filter=props.ldap_user_filter,
                 ldap_user_search_base=props.ldap_user_search_base,
                 location=props.location,
+                log_analytics_workspace=props.log_analytics_workspace,
                 resource_group_name=props.resource_group_name,
                 sre_fqdn=props.sre_fqdn,
                 storage_account_key=props.storage_account_key,
@@ -151,6 +156,7 @@ def __init__(
                 dns_server_ip=props.dns_server_ip,
                 dockerhub_credentials=props.dockerhub_credentials,
                 location=props.location,
+                log_analytics_workspace=props.log_analytics_workspace,
                 nexus_admin_password=props.nexus_admin_password,
                 resource_group_name=props.resource_group_name,
                 sre_fqdn=props.sre_fqdn,