Merge branch 'develop' into 2158-move-security-group-creation-to-pulumi

.hatch/requirements-docs.txt

@@ -1,7 +1,7 @@
 #
 # This file is autogenerated by hatch-pip-compile with Python 3.12
 #
-# - emoji==2.13.2
+# - emoji==2.14.0
 # - myst-parser==4.0.0
 # - pydata-sphinx-theme==0.15.4
 # - sphinx-togglebutton==0.3.2
@@ -28,7 +28,7 @@ docutils==0.21.2
     #   pydata-sphinx-theme
     #   sphinx
     #   sphinx-togglebutton
-emoji==2.13.2
+emoji==2.14.0
     # via hatch.envs.docs
 idna==3.10
     # via requests

.hatch/requirements-lint.txt

@@ -7,10 +7,10 @@
 # - mypy==1.11.2
 # - pandas-stubs==2.2.2.240909
 # - pydantic==2.9.2
-# - ruff==0.6.8
+# - ruff==0.6.9
 # - types-appdirs==1.4.3.5
 # - types-chevron==0.14.2.20240310
-# - types-pytz==2024.2.0.20240913
+# - types-pytz==2024.2.0.20241003
 # - types-pyyaml==6.0.12.20240917
 # - types-requests==2.32.0.20240914
 #
@@ -135,7 +135,7 @@ mypy-extensions==1.0.0
     # via
     #   black
     #   mypy
-numpy==2.1.1
+numpy==2.1.2
     # via pandas-stubs
 onigurumacffi==1.3.0
     # via ansible-navigator
@@ -222,7 +222,7 @@ referencing==0.35.1
     #   jsonschema-specifications
 resolvelib==1.0.1
     # via ansible-core
-rich==13.8.1
+rich==13.9.2
     # via
     #   ansible-lint
     #   enrich
@@ -235,22 +235,22 @@ ruamel-yaml==0.18.6
     # via ansible-lint
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-ruff==0.6.8
+ruff==0.6.9
     # via hatch.envs.lint
 subprocess-tee==0.4.2
     # via
     #   ansible-compat
     #   ansible-dev-environment
     #   ansible-lint
-tox==4.21.0
+tox==4.21.2
     # via tox-ansible
 tox-ansible==24.9.0
     # via ansible-dev-tools
 types-appdirs==1.4.3.5
     # via hatch.envs.lint
 types-chevron==0.14.2.20240310
     # via hatch.envs.lint
-types-pytz==2024.2.0.20240913
+types-pytz==2024.2.0.20241003
     # via
     #   hatch.envs.lint
     #   pandas-stubs

.hatch/requirements-test.txt

@@ -1,7 +1,7 @@
 #
 # This file is autogenerated by hatch-pip-compile with Python 3.12
 #
-# [constraints] .hatch/requirements.txt (SHA256: 69488923545009ea59b421629909ae529ee7ea3a4a3780b3a967123c9a3c7251)
+# [constraints] .hatch/requirements.txt (SHA256: f892a9714607641735b83f480e2c234b2ab8e1dffd2d59ad4188c887c06b24de)
 #
 # - appdirs==1.4.4
 # - azure-core==1.31.0
@@ -23,16 +23,16 @@
 # - chevron==0.14.0
 # - cryptography==43.0.1
 # - fqdn==1.5.1
-# - psycopg==3.2.3
-# - pulumi-azure-native==2.63.0
+# - psycopg[binary]==3.2.3
+# - pulumi-azure-native==2.64.3
 # - pulumi-azuread==5.53.4
 # - pulumi-random==4.16.6
-# - pulumi==3.134.1
+# - pulumi==3.135.1
 # - pydantic==2.9.2
 # - pyjwt[crypto]==2.9.0
 # - pytz==2024.2
 # - pyyaml==6.0.2
-# - rich==13.8.1
+# - rich==13.9.2
 # - simple-acme-dns==3.1.0
 # - typer==0.12.5
 # - websocket-client==1.8.0
@@ -291,14 +291,18 @@ psycopg==3.2.3
     # via
     #   -c .hatch/requirements.txt
     #   hatch.envs.test
-pulumi==3.134.1
+psycopg-binary==3.2.3
+    # via
+    #   -c .hatch/requirements.txt
+    #   psycopg
+pulumi==3.135.1
     # via
     #   -c .hatch/requirements.txt
     #   hatch.envs.test
     #   pulumi-azure-native
     #   pulumi-azuread
     #   pulumi-random
-pulumi-azure-native==2.63.0
+pulumi-azure-native==2.64.3
     # via
     #   -c .hatch/requirements.txt
     #   hatch.envs.test
@@ -374,7 +378,7 @@ requests-oauthlib==2.0.0
     # via
     #   -c .hatch/requirements.txt
     #   msrest
-rich==13.8.1
+rich==13.9.2
     # via
     #   -c .hatch/requirements.txt
     #   hatch.envs.test

.hatch/requirements.txt

@@ -21,16 +21,16 @@
 # - chevron==0.14.0
 # - cryptography==43.0.1
 # - fqdn==1.5.1
-# - psycopg==3.2.3
-# - pulumi-azure-native==2.63.0
+# - psycopg[binary]==3.2.3
+# - pulumi-azure-native==2.64.3
 # - pulumi-azuread==5.53.4
 # - pulumi-random==4.16.6
-# - pulumi==3.134.1
+# - pulumi==3.135.1
 # - pydantic==2.9.2
 # - pyjwt[crypto]==2.9.0
 # - pytz==2024.2
 # - pyyaml==6.0.2
-# - rich==13.8.1
+# - rich==13.9.2
 # - simple-acme-dns==3.1.0
 # - typer==0.12.5
 # - websocket-client==1.8.0
@@ -190,13 +190,15 @@ protobuf==4.25.5
     # via pulumi
 psycopg==3.2.3
     # via hatch.envs.default
-pulumi==3.134.1
+psycopg-binary==3.2.3
+    # via psycopg
+pulumi==3.135.1
     # via
     #   hatch.envs.default
     #   pulumi-azure-native
     #   pulumi-azuread
     #   pulumi-random
-pulumi-azure-native==2.63.0
+pulumi-azure-native==2.64.3
     # via hatch.envs.default
 pulumi-azuread==5.53.4
     # via hatch.envs.default
@@ -238,7 +240,7 @@ requests==2.32.3
     #   requests-oauthlib
 requests-oauthlib==2.0.0
     # via msrest
-rich==13.8.1
+rich==13.9.2
     # via
     #   hatch.envs.default
     #   typer

data_safe_haven/commands/sre.py

@@ -6,7 +6,7 @@
 
 from data_safe_haven.config import ContextManager, DSHPulumiConfig, SHMConfig, SREConfig
 from data_safe_haven.exceptions import DataSafeHavenConfigError, DataSafeHavenError
-from data_safe_haven.external import GraphApi
+from data_safe_haven.external import AzureSdk, GraphApi
 from data_safe_haven.functions import current_ip_address, ip_address_in_list
 from data_safe_haven.infrastructure import SREProjectManager
 from data_safe_haven.logging import get_logger
@@ -77,12 +77,23 @@ def deploy(
             sre_config.azure.subscription_id,
             replace=False,
         )
-        logger.info(
-            f"SRE will be deployed to subscription '[green]{sre_config.azure.subscription_id}[/]'"
-        )
         stack.add_option(
             "azure-native:tenantId", sre_config.azure.tenant_id, replace=False
         )
+        # Get SRE subscription name
+        azure_sdk = AzureSdk(subscription_name=context.subscription_name)
+        sre_subscription_name = azure_sdk.get_subscription_name(
+            sre_config.azure.subscription_id
+        )
+        stack.add_option(
+            "sre-subscription-name",
+            sre_subscription_name,
+            replace=True,
+        )
+        logger.info(
+            f"SRE will be deployed to subscription '[green]{sre_subscription_name}[/]'"
+            f" ('[bold]{sre_config.azure.subscription_id}[/]')"
+        )
         # Set Entra options
         application = graph_api.get_application_by_name(context.entra_application_name)
         if not application:
@@ -114,7 +125,18 @@ def deploy(
             shm_config.shm.fqdn,
             replace=True,
         )
-        logger.info(f"SRE will be registered in shm '[green]{shm_config.shm.fqdn}[/]'")
+        stack.add_option(
+            "shm-location",
+            shm_config.azure.location,
+            replace=True,
+        )
+        stack.add_option(
+            "shm-subscription-id",
+            shm_config.azure.subscription_id,
+            replace=True,
+        )
+        logger.info(f"SRE will be registered in SHM '[green]{shm_config.shm.fqdn}[/]'")
+        logger.info(f"SHM subscription '[green]{shm_config.azure.subscription_id}[/]'")
 
         # Deploy Azure infrastructure with Pulumi
         try:
@@ -129,7 +151,7 @@ def deploy(
             location=sre_config.azure.location,
             sre_name=sre_config.name,
             sre_stack=stack,
-            subscription_name=context.subscription_name,
+            subscription_name=sre_subscription_name,
             timezone=sre_config.sre.timezone,
         )
         manager.run()

data_safe_haven/commands/users.py

@@ -83,6 +83,10 @@ def list_users(
         # Load Pulumi config
         pulumi_config = DSHPulumiConfig.from_remote(context)
 
+        if sre not in pulumi_config.project_names:
+            msg = f"Could not load Pulumi settings for '{sre}'. Is the SRE deployed?"
+            logger.error(msg)
+            raise typer.Exit(1)
         # List users from all sources
         users = UserHandler(context, graph_api)
         users.list(sre, pulumi_config)

data_safe_haven/external/api/azure_sdk.py

@@ -436,7 +436,11 @@ def ensure_keyvault(
             )
             return key_vaults[0]
         except AzureError as exc:
-            msg = f"Failed to create key vault {key_vault_name}. Check if a key vault with the same name already exists in a deleted state."
+            msg = f"Failed to create key vault {key_vault_name}."
+            if "MissingSubscriptionRegistration" in exc.message:
+                msg += " Subscription is not registered to use the key vault resource provider. See https://learn.microsoft.com/en-us/azure/azure-resource-manager/troubleshooting/error-register-resource-provider"
+            else:
+                msg += " Check if a key vault with the same name already exists in a deleted state."
             raise DataSafeHavenAzureError(msg) from exc
 
     def ensure_keyvault_key(
@@ -774,6 +778,21 @@ def get_subscription(self, subscription_name: str) -> Subscription:
         msg = f"Could not find subscription '{subscription_name}'"
         raise DataSafeHavenValueError(msg)
 
+    def get_subscription_name(self, subscription_id: str) -> str:
+        """Get an Azure subscription name by id."""
+        try:
+            subscription_client = SubscriptionClient(self.credential())
+            subscription = subscription_client.subscriptions.get(subscription_id)
+        except ClientAuthenticationError as exc:
+            msg = "Failed to authenticate with Azure API."
+            raise DataSafeHavenAzureAPIAuthenticationError(msg) from exc
+        except AzureError as exc:
+            msg = f"Failed to get name of subscription {subscription_id}."
+            raise DataSafeHavenAzureError(msg) from exc
+
+        subscription_name: str = subscription.display_name
+        return subscription_name
+
     def import_keyvault_certificate(
         self,
         certificate_name: str,

data_safe_haven/infrastructure/programs/declarative_sre.py

@@ -54,6 +54,9 @@ def __call__(self) -> None:
         shm_admin_group_id = self.pulumi_opts.require("shm-admin-group-id")
         shm_entra_tenant_id = self.pulumi_opts.require("shm-entra-tenant-id")
         shm_fqdn = self.pulumi_opts.require("shm-fqdn")
+        shm_location = self.pulumi_opts.require("shm-location")
+        shm_subscription_id = self.pulumi_opts.require("shm-subscription-id")
+        sre_subscription_name = self.pulumi_opts.require("sre-subscription-name")
 
         # Construct DockerHubCredentials
         dockerhub_credentials = DockerHubCredentials(
@@ -149,7 +152,9 @@ def __call__(self) -> None:
                 location=self.config.azure.location,
                 resource_group_name=resource_group.name,
                 shm_fqdn=shm_fqdn,
+                shm_location=shm_location,
                 shm_resource_group_name=self.context.resource_group_name,
+                shm_subscription_id=shm_subscription_id,
                 shm_zone_name=shm_fqdn,
                 sre_name=self.config.name,
                 user_public_ip_ranges=self.config.sre.research_user_ip_addresses,
@@ -197,7 +202,7 @@ def __call__(self) -> None:
                 subnet_data_configuration=networking.subnet_data_configuration,
                 subnet_data_private=networking.subnet_data_private,
                 subscription_id=self.config.azure.subscription_id,
-                subscription_name=self.context.subscription_name,
+                subscription_name=sre_subscription_name,
                 tenant_id=self.config.azure.tenant_id,
             ),
             tags=self.tags,
@@ -369,7 +374,7 @@ def __call__(self) -> None:
                 resource_group=resource_group,
                 software_repository_hostname=user_services.software_repositories.hostname,
                 subnet_desired_state=networking.subnet_desired_state,
-                subscription_name=self.context.subscription_name,
+                subscription_name=sre_subscription_name,
             ),
         )
 
@@ -390,7 +395,7 @@ def __call__(self) -> None:
                 storage_account_data_private_user_name=data.storage_account_data_private_user_name,
                 storage_account_data_private_sensitive_name=data.storage_account_data_private_sensitive_name,
                 subnet_workspaces=networking.subnet_workspaces,
-                subscription_name=self.context.subscription_name,
+                subscription_name=sre_subscription_name,
                 virtual_network=networking.virtual_network,
                 vm_details=list(enumerate(self.config.sre.workspace_skus)),
             ),