Pulumi: Add smoke tests

data_safe_haven/external/api/azure_api.py

@@ -1084,6 +1084,42 @@ def run_remote_script(
             msg = f"Failed to run command on '{vm_name}'.\n{exc}"
             raise DataSafeHavenAzureError(msg) from exc
 
+    def run_remote_script_waiting(
+        self,
+        resource_group_name: str,
+        script: str,
+        script_parameters: dict[str, str],
+        vm_name: str,
+    ) -> str:
+        """Run a script on a remote virtual machine waiting for other scripts to complete
+
+        Returns:
+            str: The script output
+
+        Raises:
+            DataSafeHavenAzureError if running the script failed
+        """
+        while True:
+            try:
+                script_output = self.run_remote_script(
+                    resource_group_name=resource_group_name,
+                    script=script,
+                    script_parameters=script_parameters,
+                    vm_name=vm_name,
+                )
+                break
+            except DataSafeHavenAzureError as exc:
+                if all(
+                    reason not in str(exc)
+                    for reason in (
+                        "The request failed due to conflict with a concurrent request",
+                        "Run command extension execution is in progress",
+                    )
+                ):
+                    raise
+                time.sleep(5)
+        return script_output
+
     def set_blob_container_acl(
         self,
         container_name: str,

data_safe_haven/external/api/graph_api.py

@@ -361,7 +361,9 @@ def create_token_administrator(self) -> str:
         result = None
         try:
             # Load local token cache
-            local_token_cache = LocalTokenCache(pathlib.Path.home() / ".msal_cache")
+            local_token_cache = LocalTokenCache(
+                pathlib.Path.home() / f".msal_cache_{self.tenant_id}"
+            )
             # Use the Powershell application by default as this should be pre-installed
             app = PublicClientApplication(
                 authority=f"https://login.microsoftonline.com/{self.tenant_id}",

data_safe_haven/functions/strings.py

@@ -90,7 +90,7 @@ def seeded_uuid(seed: str) -> uuid.UUID:
 
 def sha256hash(input_string: str) -> str:
     """Return the SHA256 hash of a string as a string."""
-    return hashlib.sha256(str.encode(input_string, encoding="utf-8")).hexdigest()
+    return hashlib.sha256(input_string.encode("utf-8")).hexdigest()
 
 
 def truncate_tokens(tokens: Sequence[str], max_length: int) -> list[str]:

data_safe_haven/infrastructure/components/__init__.py

@@ -20,6 +20,8 @@
     CompiledDscProps,
     FileShareFile,
     FileShareFileProps,
+    FileUpload,
+    FileUploadProps,
     RemoteScript,
     RemoteScriptProps,
     SSLCertificate,
@@ -41,6 +43,8 @@
     "CompiledDscProps",
     "FileShareFile",
     "FileShareFileProps",
+    "FileUpload",
+    "FileUploadProps",
     "LinuxVMComponentProps",
     "LocalDnsRecordComponent",
     "LocalDnsRecordProps",

data_safe_haven/infrastructure/components/dynamic/__init__.py

@@ -2,7 +2,8 @@
 from .blob_container_acl import BlobContainerAcl, BlobContainerAclProps
 from .compiled_dsc import CompiledDsc, CompiledDscProps
 from .file_share_file import FileShareFile, FileShareFileProps
-from .remote_powershell import RemoteScript, RemoteScriptProps
+from .file_upload import FileUpload, FileUploadProps
+from .remote_script import RemoteScript, RemoteScriptProps
 from .ssl_certificate import SSLCertificate, SSLCertificateProps
 
 __all__ = [
@@ -14,6 +15,8 @@
     "CompiledDscProps",
     "FileShareFile",
     "FileShareFileProps",
+    "FileUpload",
+    "FileUploadProps",
     "RemoteScript",
     "RemoteScriptProps",
     "SSLCertificate",

data_safe_haven/infrastructure/components/dynamic/file_upload.py

@@ -0,0 +1,144 @@
+"""Pulumi dynamic component for running remote scripts on an Azure VM."""
+from typing import Any
+
+from pulumi import Input, Output, ResourceOptions
+from pulumi.dynamic import CreateResult, DiffResult, Resource, UpdateResult
+
+from data_safe_haven.exceptions import DataSafeHavenAzureError
+from data_safe_haven.external import AzureApi
+from data_safe_haven.functions import b64encode
+
+from .dsh_resource_provider import DshResourceProvider
+
+
+class FileUploadProps:
+    """Props for the FileUpload class"""
+
+    def __init__(
+        self,
+        file_contents: Input[str],
+        file_hash: Input[str],
+        file_permissions: Input[str],
+        file_target: Input[str],
+        subscription_name: Input[str],
+        vm_name: Input[str],
+        vm_resource_group_name: Input[str],
+        force_refresh: Input[bool] | None = None,
+    ) -> None:
+        self.file_contents = file_contents
+        self.file_hash = file_hash
+        self.file_target = file_target
+        self.file_permissions = file_permissions
+        self.force_refresh = Output.from_input(force_refresh).apply(
+            lambda force: force if force else False
+        )
+        self.subscription_name = subscription_name
+        self.vm_name = vm_name
+        self.vm_resource_group_name = vm_resource_group_name
+
+
+class FileUploadProvider(DshResourceProvider):
+    def create(self, props: dict[str, Any]) -> CreateResult:
+        """Run a remote script to create a file on a VM"""
+        outs = dict(**props)
+        azure_api = AzureApi(props["subscription_name"], disable_logging=True)
+        script_contents = f"""
+        target_dir=$(dirname "$target");
+        mkdir -p $target_dir 2> /dev/null;
+        echo $contents_b64 | base64 --decode > $target;
+        chmod {props['file_permissions']} $target;
+        if [ -f "$target" ]; then
+            echo "Wrote file to $target";
+        else
+            echo "Failed to write file to $target";
+        fi
+        """
+        script_parameters = {
+            "contents_b64": b64encode(props["file_contents"]),
+            "target": props["file_target"],
+        }
+        # Run remote script
+        script_output = azure_api.run_remote_script_waiting(
+            props["vm_resource_group_name"],
+            script_contents,
+            script_parameters,
+            props["vm_name"],
+        )
+        outs["script_output"] = "\n".join(
+            [
+                line.strip()
+                for line in script_output.replace("Enable succeeded:", "").split("\n")
+                if line
+            ]
+        )
+        if "Failed to write" in outs["script_output"]:
+            raise DataSafeHavenAzureError(outs["script_output"])
+        return CreateResult(
+            f"FileUpload-{props['file_hash']}",
+            outs=outs,
+        )
+
+    def delete(self, id_: str, props: dict[str, Any]) -> None:
+        """Delete the remote file from the VM"""
+        # Use `id` as a no-op to avoid ARG002 while maintaining function signature
+        id(id_)
+        azure_api = AzureApi(props["subscription_name"], disable_logging=True)
+        script_contents = """
+        rm -f "$target";
+        echo "Removed file at $target";
+        """
+        script_parameters = {
+            "target": props["file_target"],
+        }
+        # Run remote script
+        azure_api.run_remote_script_waiting(
+            props["vm_resource_group_name"],
+            script_contents,
+            script_parameters,
+            props["vm_name"],
+        )
+
+    def diff(
+        self,
+        id_: str,
+        old_props: dict[str, Any],
+        new_props: dict[str, Any],
+    ) -> DiffResult:
+        """Calculate diff between old and new state"""
+        # Use `id` as a no-op to avoid ARG002 while maintaining function signature
+        id(id_)
+        if new_props["force_refresh"]:
+            return DiffResult(
+                changes=True,
+                replaces=list(new_props.keys()),
+                stables=[],
+                delete_before_replace=False,
+            )
+        return self.partial_diff(old_props, new_props, [])
+
+    def update(
+        self,
+        id_: str,
+        old_props: dict[str, Any],
+        new_props: dict[str, Any],
+    ) -> UpdateResult:
+        """Updating is creating without the need to delete."""
+        # Use `id` as a no-op to avoid ARG002 while maintaining function signature
+        id((id_, old_props))
+        updated = self.create(new_props)
+        return UpdateResult(outs=updated.outs)
+
+
+class FileUpload(Resource):
+    script_output: Output[str]
+    _resource_type_name = "dsh:common:FileUpload"  # set resource type
+
+    def __init__(
+        self,
+        name: str,
+        props: FileUploadProps,
+        opts: ResourceOptions | None = None,
+    ):
+        super().__init__(
+            FileUploadProvider(), name, {"script_output": None, **vars(props)}, opts
+        )

data_safe_haven/infrastructure/stacks/declarative_sre.py

@@ -253,6 +253,7 @@ def run(self) -> None:
                 storage_account_data_private_user_name=data.storage_account_data_private_user_name,
                 storage_account_data_private_sensitive_name=data.storage_account_data_private_sensitive_name,
                 subnet_workspaces=networking.subnet_workspaces,
+                subscription_name=self.cfg.subscription_name,
                 virtual_network_resource_group=networking.resource_group,
                 virtual_network=networking.virtual_network,
                 vm_details=list(enumerate(self.cfg.sres[self.sre_name].workspace_skus)),

data_safe_haven/infrastructure/stacks/sre/workspaces.py

@@ -1,3 +1,4 @@
+import pathlib
 from collections.abc import Mapping
 from typing import Any
 
@@ -14,10 +15,13 @@
     get_name_from_vnet,
 )
 from data_safe_haven.infrastructure.components import (
+    FileUpload,
+    FileUploadProps,
     LinuxVMComponentProps,
     VMComponent,
 )
 from data_safe_haven.resources import resources_path
+from data_safe_haven.utility import FileReader
 
 
 class SREWorkspacesProps:
@@ -43,6 +47,7 @@ def __init__(
         storage_account_data_private_user_name: Input[str],
         storage_account_data_private_sensitive_name: Input[str],
         subnet_workspaces: Input[network.GetSubnetResult],
+        subscription_name: Input[str],
         virtual_network_resource_group: Input[resources.ResourceGroup],
         virtual_network: Input[network.VirtualNetwork],
         vm_details: list[tuple[int, str]],  # this must *not* be passed as an Input[T]
@@ -69,6 +74,7 @@ def __init__(
         self.storage_account_data_private_sensitive_name = (
             storage_account_data_private_sensitive_name
         )
+        self.subscription_name = subscription_name
         self.virtual_network_name = Output.from_input(virtual_network).apply(
             get_name_from_vnet
         )
@@ -161,7 +167,7 @@ def __init__(
         ]
 
         # Get details for each deployed VM
-        vm_outputs = [
+        vm_outputs: list[dict[str, Any]] = [
             {
                 "ip_address": vm.ip_address_private,
                 "name": vm.vm_name,
@@ -170,6 +176,36 @@ def __init__(
             for vm in vms
         ]
 
+        # Upload smoke tests
+        mustache_values = {
+            "check_uninstallable_packages": "0",
+        }
+        file_uploads = [
+            (FileReader(resources_path / "workspace" / "run_all_tests.bats"), "0444")
+        ]
+        for test_file in pathlib.Path(resources_path / "workspace").glob("test*"):
+            file_uploads.append((FileReader(test_file), "0444"))
+        for vm, vm_output in zip(vms, vm_outputs, strict=True):
+            outputs: dict[str, Output[str]] = {}
+            for file_upload, file_permissions in file_uploads:
+                file_smoke_test = FileUpload(
+                    replace_separators(f"{self._name}_file_{file_upload.name}", "_"),
+                    FileUploadProps(
+                        file_contents=file_upload.file_contents(
+                            mustache_values=mustache_values
+                        ),
+                        file_hash=file_upload.sha256(),
+                        file_permissions=file_permissions,
+                        file_target=f"/opt/tests/{file_upload.name}",
+                        subscription_name=props.subscription_name,
+                        vm_name=vm.vm_name,
+                        vm_resource_group_name=resource_group.name,
+                    ),
+                    opts=child_opts,
+                )
+                outputs[file_upload.name] = file_smoke_test.script_output
+            vm_output["file_uploads"] = outputs
+
         # Register outputs
         self.resource_group = resource_group
 

data_safe_haven/resources/software_repositories/allowlists/cran.allowlist

@@ -1,4 +1,5 @@
 DBI
+MASS
 RPostgres
 Rcpp
 bit

data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist

@@ -15,6 +15,7 @@ pyodbc
 pyparsing
 python-dateutil
 pytz
+scikit-learn
 six
 typing-extensions
 tzdata