Merge pull request alan-turing-institute#2160 from jemrobinson/2158-m…

…ove-security-group-creation-to-pulumi

Move security group creation to Pulumi

.hatch/requirements-test.txt

@@ -1,7 +1,7 @@
 #
 # This file is autogenerated by hatch-pip-compile with Python 3.12
 #
-# [constraints] .hatch/requirements.txt (SHA256: 697cb5b4ddc1cb9481ae4847d33cf407119c028729be950240b30eaddd8c294e)
+# [constraints] .hatch/requirements.txt (SHA256: f892a9714607641735b83f480e2c234b2ab8e1dffd2d59ad4188c887c06b24de)
 #
 # - appdirs==1.4.4
 # - azure-core==1.31.0
@@ -25,6 +25,7 @@
 # - fqdn==1.5.1
 # - psycopg[binary]==3.2.3
 # - pulumi-azure-native==2.64.3
+# - pulumi-azuread==5.53.4
 # - pulumi-random==4.16.6
 # - pulumi==3.135.1
 # - pydantic==2.9.2
@@ -274,6 +275,7 @@ parver==0.5
     # via
     #   -c .hatch/requirements.txt
     #   pulumi-azure-native
+    #   pulumi-azuread
     #   pulumi-random
 pluggy==1.5.0
     # via pytest
@@ -298,11 +300,16 @@ pulumi==3.135.1
     #   -c .hatch/requirements.txt
     #   hatch.envs.test
     #   pulumi-azure-native
+    #   pulumi-azuread
     #   pulumi-random
 pulumi-azure-native==2.64.3
     # via
     #   -c .hatch/requirements.txt
     #   hatch.envs.test
+pulumi-azuread==5.53.4
+    # via
+    #   -c .hatch/requirements.txt
+    #   hatch.envs.test
 pulumi-random==4.16.6
     # via
     #   -c .hatch/requirements.txt
@@ -381,6 +388,7 @@ semver==2.13.0
     #   -c .hatch/requirements.txt
     #   pulumi
     #   pulumi-azure-native
+    #   pulumi-azuread
     #   pulumi-random
 shellingham==1.5.4
     # via

.hatch/requirements.txt

@@ -23,6 +23,7 @@
 # - fqdn==1.5.1
 # - psycopg[binary]==3.2.3
 # - pulumi-azure-native==2.64.3
+# - pulumi-azuread==5.53.4
 # - pulumi-random==4.16.6
 # - pulumi==3.135.1
 # - pydantic==2.9.2
@@ -181,6 +182,7 @@ oauthlib==3.2.2
 parver==0.5
     # via
     #   pulumi-azure-native
+    #   pulumi-azuread
     #   pulumi-random
 portalocker==2.10.1
     # via msal-extensions
@@ -194,9 +196,12 @@ pulumi==3.135.1
     # via
     #   hatch.envs.default
     #   pulumi-azure-native
+    #   pulumi-azuread
     #   pulumi-random
 pulumi-azure-native==2.64.3
     # via hatch.envs.default
+pulumi-azuread==5.53.4
+    # via hatch.envs.default
 pulumi-random==4.16.6
     # via hatch.envs.default
 pycparser==2.22
@@ -243,6 +248,7 @@ semver==2.13.0
     # via
     #   pulumi
     #   pulumi-azure-native
+    #   pulumi-azuread
     #   pulumi-random
 shellingham==1.5.4
     # via typer

data_safe_haven/commands/sre.py

@@ -91,7 +91,23 @@ def deploy(
             replace=True,
         )
         logger.info(
-            f"SRE will be deployed to subscription '[green]{sre_config.azure.subscription_id}[/]', '[green]{sre_subscription_name}[/]'"
+            f"SRE will be deployed to subscription '[green]{sre_subscription_name}[/]'"
+            f" ('[bold]{sre_config.azure.subscription_id}[/]')"
+        )
+        # Set Entra options
+        application = graph_api.get_application_by_name(context.entra_application_name)
+        if not application:
+            msg = f"No Entra application '{context.entra_application_name}' was found. Please redeploy your SHM."
+            raise DataSafeHavenConfigError(msg)
+        stack.add_option("azuread:clientId", application.get("appId", ""), replace=True)
+        if not context.entra_application_secret:
+            msg = f"No Entra application secret '{context.entra_application_secret_name}' was found. Please redeploy your SHM."
+            raise DataSafeHavenConfigError(msg)
+        stack.add_secret(
+            "azuread:clientSecret", context.entra_application_secret, replace=True
+        )
+        stack.add_option(
+            "azuread:tenantId", shm_config.shm.entra_tenant_id, replace=True
         )
         # Load SHM outputs
         stack.add_option(

data_safe_haven/config/context.py

@@ -9,46 +9,56 @@
 
 from data_safe_haven import __version__
 from data_safe_haven.directories import config_dir
+from data_safe_haven.exceptions import DataSafeHavenAzureError
 from data_safe_haven.external import AzureSdk
 from data_safe_haven.functions import alphanumeric
 from data_safe_haven.serialisers import ContextBase
 from data_safe_haven.types import AzureSubscriptionName, EntraGroupName, SafeString
 
 
 class Context(ContextBase, BaseModel, validate_assignment=True):
+    """Context for a Data Safe Haven deployment."""
+
+    entra_application_kvsecret_name: ClassVar[str] = "pulumi-deployment-secret"
+    entra_application_secret_name: ClassVar[str] = "Pulumi Deployment Secret"
+    pulumi_encryption_key_name: ClassVar[str] = "pulumi-encryption-key"
+    pulumi_storage_container_name: ClassVar[str] = "pulumi"
+    storage_container_name: ClassVar[str] = "config"
+
     admin_group_name: EntraGroupName
     description: str
     name: SafeString
     subscription_name: AzureSubscriptionName
-    storage_container_name: ClassVar[str] = "config"
-    pulumi_storage_container_name: ClassVar[str] = "pulumi"
-    pulumi_encryption_key_name: ClassVar[str] = "pulumi-encryption-key"
 
     _pulumi_encryption_key = None
+    _entra_application_secret = None
 
     @property
-    def tags(self) -> dict[str, str]:
-        return {
-            "description": self.description,
-            "project": "Data Safe Haven",
-            "shm_name": self.name,
-            "version": __version__,
-        }
+    def entra_application_name(self) -> str:
+        return f"Data Safe Haven ({self.description}) Pulumi Service Principal"
 
     @property
-    def work_directory(self) -> Path:
-        return config_dir() / self.name
-
-    @property
-    def resource_group_name(self) -> str:
-        return f"shm-{self.name}-rg"
-
-    @property
-    def storage_account_name(self) -> str:
-        # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-name
-        #   Storage account names must be between 3 and 24 characters in length and may
-        #   contain numbers and lowercase letters only.
-        return f"shm{alphanumeric(self.name)[:21]}"
+    def entra_application_secret(self) -> str:
+        if not self._entra_application_secret:
+            azure_sdk = AzureSdk(subscription_name=self.subscription_name)
+            try:
+                application_secret = azure_sdk.get_keyvault_secret(
+                    secret_name=self.entra_application_kvsecret_name,
+                    key_vault_name=self.key_vault_name,
+                )
+                self._entra_application_secret = application_secret
+            except DataSafeHavenAzureError:
+                return ""
+        return self._entra_application_secret
+
+    @entra_application_secret.setter
+    def entra_application_secret(self, application_secret: str) -> None:
+        azure_sdk = AzureSdk(subscription_name=self.subscription_name)
+        azure_sdk.set_keyvault_secret(
+            secret_name=self.entra_application_kvsecret_name,
+            secret_value=application_secret,
+            key_vault_name=self.key_vault_name,
+        )
 
     @property
     def key_vault_name(self) -> str:
@@ -83,5 +93,29 @@ def pulumi_encryption_key_version(self) -> str:
     def pulumi_secrets_provider_url(self) -> str:
         return f"azurekeyvault://{self.key_vault_name}.vault.azure.net/keys/{self.pulumi_encryption_key_name}/{self.pulumi_encryption_key_version}"
 
+    @property
+    def resource_group_name(self) -> str:
+        return f"shm-{self.name}-rg"
+
+    @property
+    def storage_account_name(self) -> str:
+        # https://learn.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-name
+        #   Storage account names must be between 3 and 24 characters in length and may
+        #   contain numbers and lowercase letters only.
+        return f"shm{alphanumeric(self.name)[:21]}"
+
+    @property
+    def tags(self) -> dict[str, str]:
+        return {
+            "description": self.description,
+            "project": "Data Safe Haven",
+            "shm_name": self.name,
+            "version": __version__,
+        }
+
+    @property
+    def work_directory(self) -> Path:
+        return config_dir() / self.name
+
     def to_yaml(self) -> str:
         return yaml.dump(self.model_dump(), indent=2)

data_safe_haven/config/dsh_pulumi_config.py

@@ -12,6 +12,7 @@ class DSHPulumiConfig(AzureSerialisableModel):
 
     config_type: ClassVar[str] = "Pulumi"
     default_filename: ClassVar[str] = "pulumi.yaml"
+
     encrypted_key: str | None
     projects: dict[str, DSHPulumiProject]
 

data_safe_haven/config/shm_config.py

@@ -12,8 +12,11 @@
 
 
 class SHMConfig(AzureSerialisableModel):
+    """Serialisable config for a Data Safe Haven management component."""
+
     config_type: ClassVar[str] = "SHMConfig"
     default_filename: ClassVar[str] = "shm.yaml"
+
     azure: ConfigSectionAzure
     shm: ConfigSectionSHM
 

data_safe_haven/config/sre_config.py

@@ -23,8 +23,11 @@ def sre_config_name(sre_name: str) -> str:
 
 
 class SREConfig(AzureSerialisableModel):
+    """Serialisable config for a secure research environment component."""
+
     config_type: ClassVar[str] = "SREConfig"
     default_filename: ClassVar[str] = "sre.yaml"
+
     azure: ConfigSectionAzure
     description: str
     dockerhub: ConfigSectionDockerHub

data_safe_haven/external/api/azure_sdk.py

@@ -14,7 +14,7 @@
 )
 from azure.keyvault.certificates import CertificateClient, KeyVaultCertificate
 from azure.keyvault.keys import KeyClient, KeyVaultKey
-from azure.keyvault.secrets import SecretClient
+from azure.keyvault.secrets import KeyVaultSecret, SecretClient
 from azure.mgmt.compute.v2021_07_01 import ComputeManagementClient
 from azure.mgmt.compute.v2021_07_01.models import (
     ResourceSkuCapabilities,
@@ -451,7 +451,7 @@ def ensure_keyvault_key(
         """Ensure that a key exists in the KeyVault
 
         Returns:
-            str: The key ID
+            KeyVaultKey: The key
 
         Raises:
             DataSafeHavenAzureError if the existence of the key could not be verified
@@ -476,7 +476,7 @@ def ensure_keyvault_key(
             )
             return key
         except AzureError as exc:
-            msg = f"Failed to create key {key_name}."
+            msg = f"Failed to create key '{key_name}' in KeyVault '{key_vault_name}'."
             raise DataSafeHavenAzureError(msg) from exc
 
     def ensure_managed_identity(
@@ -693,7 +693,7 @@ def get_keyvault_secret(self, key_vault_name: str, secret_name: str) -> str:
             credential=self.credential(AzureSdkCredentialScope.KEY_VAULT),
             vault_url=f"https://{key_vault_name}.vault.azure.net",
         )
-        # Ensure that secret exists
+        # Get secret if it exists
         try:
             secret = secret_client.get_secret(secret_name)
             if secret.value:
@@ -1302,6 +1302,38 @@ def set_blob_container_acl(
             msg = f"Failed to set ACL '{desired_acl}' on container '{container_name}'."
             raise DataSafeHavenAzureError(msg) from exc
 
+    def set_keyvault_secret(
+        self,
+        secret_name: str,
+        secret_value: str,
+        key_vault_name: str,
+    ) -> KeyVaultSecret:
+        """Ensure that a secret exists in the KeyVault
+
+        Returns:
+            KeyVaultSecret: The secret
+
+        Raises:
+            DataSafeHavenAzureError if the secret could not be set
+        """
+        try:
+            # Connect to Azure clients
+            secret_client = SecretClient(
+                credential=self.credential(AzureSdkCredentialScope.KEY_VAULT),
+                vault_url=f"https://{key_vault_name}.vault.azure.net",
+            )
+
+            # Set secret to given value
+            self.logger.debug(f"Setting secret [green]{secret_name}[/]...")
+            secret = secret_client.set_secret(secret_name, secret_value)
+            self.logger.info(f"Set secret [green]{secret_name}[/].")
+            return secret
+        except AzureError as exc:
+            msg = (
+                f"Failed to set secret '{secret_name}' in KeyVault '{key_vault_name}'."
+            )
+            raise DataSafeHavenAzureError(msg) from exc
+
     def storage_exists(
         self,
         storage_account_name: str,