Merge remote-tracking branch 'origin/python-migration' into login_flow

.devcontainer/Dockerfile

@@ -6,7 +6,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 
 # Set package versions
 ARG AZURE_CLI_VERSION="2.42.0"
-ARG PWSH_VERSION="7.3.2"
+ARG PWSH_VERSION="7.3.6"
 
 # Set up TARGETARCH variable to use to pull the right binaries for the current architecture.
 ARG TARGETARCH

.devcontainer/python/Dockerfile

@@ -3,9 +3,7 @@ FROM python:${VARIANT}-buster
 
 # Set package versions
 ARG AZURE_CLI_VERSION="2.42.0"
-ARG PWSH_VERSION="7.2.6"
-ARG PULUMI_VERSION="3.45.0"
-ARG POETRY_VERSION="1.2.2"
+ARG PULUMI_VERSION="3.80.0"
 
 RUN apt-get update \
     && export DEBIAN_FRONTEND=noninteractive \
@@ -22,21 +20,7 @@ ARG TARGETARCH
 # Standard install method currently does not support ARM64
 # Use pip instead - https://github.com/Azure/azure-cli/issues/22875
 RUN pip3 install azure-cli==${AZURE_CLI_VERSION}
-
-# Install Powershell
-# Pull different binaries from Github depending on system architecture
-# The standard APT method currently only works for `amd64`
- RUN if [ "${TARGETARCH}" = "arm64" ]; \
-   then \
-     DEBARCH="arm64"; \
-   else \
-     DEBARCH="x86"; \
-   fi; \
-   curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v${PWSH_VERSION}/powershell-${PWSH_VERSION}-linux-$DEBARCH.tar.gz \
-   && mkdir -p /opt/microsoft/powershell/7 \
-   && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 \
-   && chmod +x /opt/microsoft/powershell/7/pwsh \
-   && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh
+RUN pip3 install hatch
 
 # Create non-root user and give them sudo access
 ARG USERNAME=deploydsh
@@ -56,17 +40,5 @@ USER $USERNAME
 COPY ./docs/requirements.txt /build/requirements.txt
 RUN pip3 install -r /build/requirements.txt
 
-# Install/check needed powershell modules
-COPY ./deployment/CheckRequirements.ps1 /build/CheckRequirements.ps1
-COPY ./deployment/common/Logging.psm1 /build/common/Logging.psm1
-RUN pwsh -Command "& {Set-PSRepository -Name PSGallery -InstallationPolicy Trusted}" \
-  && pwsh -File /build/CheckRequirements.ps1 -InstallMissing \
-  && sudo rm -rf /build/
-
 # Set PATH for pulumi - pulumi installed as feature to work round installing as root
 ENV PATH=$PATH:/home/${USERNAME}/.pulumi/bin
-
-# Install poetry
-# Respects environment variables $POETRY_HOME and $POETRY_VERSION
-RUN curl -sSL https://install.python-poetry.org | python3 - \
-  && /home/${USERNAME}/.local/bin/poetry config virtualenvs.in-project true

.devcontainer/python/devcontainer.json

@@ -2,14 +2,8 @@
 {
     "name": "Turing Data Safe Haven - Pulumi",
     "build": {
-        "context": "..",
-        "dockerfile": "Dockerfile",
-        "args": {
-            "POETRY_VERSION": "1.2.2",
-            "VARIANT": "3.10",
-            "PWSH_VERSION": "7.2.6",
-            "AZURE_CLI_VERSION": "2.42.0"
-        }
+        "context": "../..",
+        "dockerfile": "Dockerfile"
     },
     "settings": {
         "terminal.integrated.defaultProfile.linux": "bash"
@@ -24,7 +18,7 @@
     "remoteUser": "deploydsh",
     "features": {
         "ghcr.io/devcontainers-contrib/features/pulumi:1": {
-            "version": "3.45.0",
+            "version": "3.80.0",
             "bashCompletion": false
         }
     }

CODEOWNERS

@@ -10,5 +10,5 @@
 # We only plan to require code owner review for
 # main and other branches that may be deployed from.
 # Note: /dir/ applies to directory and all subdirectories
-/deployment/      @martintoreilly @jemrobinson @JimMadge
+/deployment/      @martintoreilly @jemrobinson @JimMadge @craddm
 /docs/            @martintoreilly @jemrobinson @JimMadge @craddm @edwardchalstrey1

VERSIONING.md

@@ -64,6 +64,7 @@ We usually deploy the latest available version of the Data Safe Haven for each o
 | December 2021    | DSG 2021-12                   | [v3.3.1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.3.1)           |
 | December 2022    | DSG 2022-12                   | [v4.0.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.2)           |
 | February 2023    | DSG 2023-02                   | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3)           |
+| May 2023         | DSG 2023-05                   | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3)           |
 
 ## Versions that have undergone formal security evaluation
 

data_safe_haven/README.md

@@ -77,3 +77,49 @@ where you must specify the usernames for each user you want to add to this SRE
 ```bash
 > dsh teardown backend
 ```
+
+## Code structure
+
+- administration
+    - this is where we keep utility commands for adminstrators of a deployed DSH
+    - eg. "add a user"; "remove a user from an SRE"
+- backend
+    - in order to use the Pulumi Azure backend we need a KeyVault, Identity and Storage Account
+    - this code deploys those resources to bootstrap the rest of the Pulumi-based code
+- commands
+    - the main `dsh` command line entrypoint lives in `cli.py`
+    - the subsidiary `typer` command line entrypoints (eg. `dsh deploy shm`) live here
+- config
+    - serialises and deserialises a config file from Azure
+    - `backend_settings` manages basic settings related to the Azure backend: arguably this could/should live in `backend`
+- exceptions
+    - definitions of a Python exception hierarchy
+- external
+    - Python wrappers around:
+        - APIs: Azure Python SDK, Azure CLI, Graph API
+        - Azure interfaces: CLI authentication, container instances, fileshares, available IP addresses in a subnet, databases
+        - Utility for running scripts on databases
+- functions
+    - Various functions that don't fit anywhere else
+    - string manipulation, type conversions, validators, lists of allowed external FQDNs
+- infrastructure
+    - Management of the Pulumi stack, which handles passing the correct backend options
+    - common
+        - common Pulumi transformations, enums and IP address ranges
+    - components
+        - composite
+            - a logical group of existing Pulumi components that is used in several places
+        - dynamic
+            - a custom component to implement some functionality that is not natively supported
+        - wrapped
+            - thin wrappers around Pulumi resources to expose additional methods/attributes
+    - stacks
+        - definitions of the `shm` and `sre` stacks
+- provisioning
+    - all configuration options that is currently done outside Pulumi
+    - eg. Initialise the Guacamole database, reboot some VMs, create security groups on domain controller
+    - in the future this could be replaced by better orchestration options (eg. Ansible) or moved into Pulumi
+- resources
+    - configuration files and templates used by Pulumi (e.g. cloud-init configs, Caddyfiles etc.)
+- utility
+    - Useful classes: logging, file reading, types

data_safe_haven/config/backend_settings.py

@@ -88,7 +88,7 @@ def name(self) -> str:
         if not self._name:
             msg = (
                 "Data Safe Haven deployment name not provided:"
-                " use '[bright_cyan]--deployment-name[/]' / '[green]-d[/]' to do so."
+                " use '[bright_cyan]--name[/]' / '[green]-n[/]' to do so."
             )
             raise DataSafeHavenParameterError(msg)
         return self._name

data_safe_haven/config/config.py

@@ -12,7 +12,11 @@
 from yaml.parser import ParserError
 
 from data_safe_haven import __version__
-from data_safe_haven.exceptions import DataSafeHavenAzureError, DataSafeHavenConfigError
+from data_safe_haven.exceptions import (
+    DataSafeHavenAzureError,
+    DataSafeHavenConfigError,
+    DataSafeHavenParameterError,
+)
 from data_safe_haven.external import AzureApi
 from data_safe_haven.functions import (
     alphanumeric,
@@ -352,7 +356,12 @@ def __init__(self) -> None:
         self.sres: dict[str, ConfigSectionSRE] = defaultdict(ConfigSectionSRE)
         # Read backend settings
         settings = BackendSettings()
-        self.name = settings.name
+        # Check if backend exists and was loaded
+        try:
+            self.name = settings.name
+        except DataSafeHavenParameterError as exc:
+            msg = "Data Safe Haven has not been initialised: run '[bright_cyan]dsh init[/]' before continuing."
+            raise DataSafeHavenConfigError(msg) from exc
         self.subscription_name = settings.subscription_name
         self.azure.location = settings.location
         self.azure.admin_group_id = settings.admin_group_id

data_safe_haven/external/api/azure_api.py

@@ -1084,6 +1084,42 @@ def run_remote_script(
             msg = f"Failed to run command on '{vm_name}'.\n{exc}"
             raise DataSafeHavenAzureError(msg) from exc
 
+    def run_remote_script_waiting(
+        self,
+        resource_group_name: str,
+        script: str,
+        script_parameters: dict[str, str],
+        vm_name: str,
+    ) -> str:
+        """Run a script on a remote virtual machine waiting for other scripts to complete
+
+        Returns:
+            str: The script output
+
+        Raises:
+            DataSafeHavenAzureError if running the script failed
+        """
+        while True:
+            try:
+                script_output = self.run_remote_script(
+                    resource_group_name=resource_group_name,
+                    script=script,
+                    script_parameters=script_parameters,
+                    vm_name=vm_name,
+                )
+                break
+            except DataSafeHavenAzureError as exc:
+                if all(
+                    reason not in str(exc)
+                    for reason in (
+                        "The request failed due to conflict with a concurrent request",
+                        "Run command extension execution is in progress",
+                    )
+                ):
+                    raise
+                time.sleep(5)
+        return script_output
+
     def set_blob_container_acl(
         self,
         container_name: str,

data_safe_haven/external/api/graph_api.py

@@ -361,7 +361,9 @@ def create_token_administrator(self) -> str:
         result = None
         try:
             # Load local token cache
-            local_token_cache = LocalTokenCache(pathlib.Path.home() / ".msal_cache")
+            local_token_cache = LocalTokenCache(
+                pathlib.Path.home() / f".msal_cache_{self.tenant_id}"
+            )
             # Use the Powershell application by default as this should be pre-installed
             app = PublicClientApplication(
                 authority=f"https://login.microsoftonline.com/{self.tenant_id}",

data_safe_haven/functions/strings.py

@@ -90,7 +90,7 @@ def seeded_uuid(seed: str) -> uuid.UUID:
 
 def sha256hash(input_string: str) -> str:
     """Return the SHA256 hash of a string as a string."""
-    return hashlib.sha256(str.encode(input_string, encoding="utf-8")).hexdigest()
+    return hashlib.sha256(input_string.encode("utf-8")).hexdigest()
 
 
 def truncate_tokens(tokens: Sequence[str], max_length: int) -> list[str]:

data_safe_haven/infrastructure/components/__init__.py

@@ -20,6 +20,8 @@
     CompiledDscProps,
     FileShareFile,
     FileShareFileProps,
+    FileUpload,
+    FileUploadProps,
     RemoteScript,
     RemoteScriptProps,
     SSLCertificate,
@@ -41,6 +43,8 @@
     "CompiledDscProps",
     "FileShareFile",
     "FileShareFileProps",
+    "FileUpload",
+    "FileUploadProps",
     "LinuxVMComponentProps",
     "LocalDnsRecordComponent",
     "LocalDnsRecordProps",

data_safe_haven/infrastructure/components/dynamic/__init__.py

@@ -2,7 +2,8 @@
 from .blob_container_acl import BlobContainerAcl, BlobContainerAclProps
 from .compiled_dsc import CompiledDsc, CompiledDscProps
 from .file_share_file import FileShareFile, FileShareFileProps
-from .remote_powershell import RemoteScript, RemoteScriptProps
+from .file_upload import FileUpload, FileUploadProps
+from .remote_script import RemoteScript, RemoteScriptProps
 from .ssl_certificate import SSLCertificate, SSLCertificateProps
 
 __all__ = [
@@ -14,6 +15,8 @@
     "CompiledDscProps",
     "FileShareFile",
     "FileShareFileProps",
+    "FileUpload",
+    "FileUploadProps",
     "RemoteScript",
     "RemoteScriptProps",
     "SSLCertificate",