diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 85ab92fd66..4a852d8499 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # Set package versions ARG AZURE_CLI_VERSION="2.42.0" -ARG PWSH_VERSION="7.3.2" +ARG PWSH_VERSION="7.3.6" # Set up TARGETARCH variable to use to pull the right binaries for the current architecture. ARG TARGETARCH diff --git a/.devcontainer/python/Dockerfile b/.devcontainer/python/Dockerfile index c40f477b70..a28ef01333 100644 --- a/.devcontainer/python/Dockerfile +++ b/.devcontainer/python/Dockerfile @@ -3,9 +3,7 @@ FROM python:${VARIANT}-buster # Set package versions ARG AZURE_CLI_VERSION="2.42.0" -ARG PWSH_VERSION="7.2.6" -ARG PULUMI_VERSION="3.45.0" -ARG POETRY_VERSION="1.2.2" +ARG PULUMI_VERSION="3.80.0" RUN apt-get update \ && export DEBIAN_FRONTEND=noninteractive \ @@ -22,21 +20,7 @@ ARG TARGETARCH # Standard install method currently does not support ARM64 # Use pip instead - https://github.com/Azure/azure-cli/issues/22875 RUN pip3 install azure-cli==${AZURE_CLI_VERSION} - -# Install Powershell -# Pull different binaries from Github depending on system architecture -# The standard APT method currently only works for `amd64` - RUN if [ "${TARGETARCH}" = "arm64" ]; \ - then \ - DEBARCH="arm64"; \ - else \ - DEBARCH="x86"; \ - fi; \ - curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v${PWSH_VERSION}/powershell-${PWSH_VERSION}-linux-$DEBARCH.tar.gz \ - && mkdir -p /opt/microsoft/powershell/7 \ - && tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 \ - && chmod +x /opt/microsoft/powershell/7/pwsh \ - && ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh +RUN pip3 install hatch # Create non-root user and give them sudo access ARG USERNAME=deploydsh @@ -56,17 +40,5 @@ USER $USERNAME COPY ./docs/requirements.txt /build/requirements.txt RUN pip3 install -r /build/requirements.txt -# Install/check needed powershell modules -COPY ./deployment/CheckRequirements.ps1 /build/CheckRequirements.ps1 -COPY ./deployment/common/Logging.psm1 /build/common/Logging.psm1 -RUN pwsh -Command "& {Set-PSRepository -Name PSGallery -InstallationPolicy Trusted}" \ - && pwsh -File /build/CheckRequirements.ps1 -InstallMissing \ - && sudo rm -rf /build/ - # Set PATH for pulumi - pulumi installed as feature to work round installing as root ENV PATH=$PATH:/home/${USERNAME}/.pulumi/bin - -# Install poetry -# Respects environment variables $POETRY_HOME and $POETRY_VERSION -RUN curl -sSL https://install.python-poetry.org | python3 - \ - && /home/${USERNAME}/.local/bin/poetry config virtualenvs.in-project true \ No newline at end of file diff --git a/.devcontainer/python/devcontainer.json b/.devcontainer/python/devcontainer.json index 191bff51d5..6bd56275ee 100644 --- a/.devcontainer/python/devcontainer.json +++ b/.devcontainer/python/devcontainer.json @@ -2,14 +2,8 @@ { "name": "Turing Data Safe Haven - Pulumi", "build": { - "context": "..", - "dockerfile": "Dockerfile", - "args": { - "POETRY_VERSION": "1.2.2", - "VARIANT": "3.10", - "PWSH_VERSION": "7.2.6", - "AZURE_CLI_VERSION": "2.42.0" - } + "context": "../..", + "dockerfile": "Dockerfile" }, "settings": { "terminal.integrated.defaultProfile.linux": "bash" @@ -24,7 +18,7 @@ "remoteUser": "deploydsh", "features": { "ghcr.io/devcontainers-contrib/features/pulumi:1": { - "version": "3.45.0", + "version": "3.80.0", "bashCompletion": false } } diff --git a/CODEOWNERS b/CODEOWNERS index 411890bbaa..85f3c50e40 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -10,5 +10,5 @@ # We only plan to require code owner review for # main and other branches that may be deployed from. # Note: /dir/ applies to directory and all subdirectories -/deployment/ @martintoreilly @jemrobinson @JimMadge +/deployment/ @martintoreilly @jemrobinson @JimMadge @craddm /docs/ @martintoreilly @jemrobinson @JimMadge @craddm @edwardchalstrey1 diff --git a/VERSIONING.md b/VERSIONING.md index 203a10a1ff..e072045b15 100644 --- a/VERSIONING.md +++ b/VERSIONING.md @@ -64,6 +64,7 @@ We usually deploy the latest available version of the Data Safe Haven for each o | December 2021 | DSG 2021-12 | [v3.3.1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.3.1) | | December 2022 | DSG 2022-12 | [v4.0.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.2) | | February 2023 | DSG 2023-02 | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3) | +| May 2023 | DSG 2023-05 | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3) | ## Versions that have undergone formal security evaluation diff --git a/data_safe_haven/README.md b/data_safe_haven/README.md index 0f6f1b0179..83feed0443 100644 --- a/data_safe_haven/README.md +++ b/data_safe_haven/README.md @@ -77,3 +77,49 @@ where you must specify the usernames for each user you want to add to this SRE ```bash > dsh teardown backend ``` + +## Code structure + +- administration + - this is where we keep utility commands for adminstrators of a deployed DSH + - eg. "add a user"; "remove a user from an SRE" +- backend + - in order to use the Pulumi Azure backend we need a KeyVault, Identity and Storage Account + - this code deploys those resources to bootstrap the rest of the Pulumi-based code +- commands + - the main `dsh` command line entrypoint lives in `cli.py` + - the subsidiary `typer` command line entrypoints (eg. `dsh deploy shm`) live here +- config + - serialises and deserialises a config file from Azure + - `backend_settings` manages basic settings related to the Azure backend: arguably this could/should live in `backend` +- exceptions + - definitions of a Python exception hierarchy +- external + - Python wrappers around: + - APIs: Azure Python SDK, Azure CLI, Graph API + - Azure interfaces: CLI authentication, container instances, fileshares, available IP addresses in a subnet, databases + - Utility for running scripts on databases +- functions + - Various functions that don't fit anywhere else + - string manipulation, type conversions, validators, lists of allowed external FQDNs +- infrastructure + - Management of the Pulumi stack, which handles passing the correct backend options + - common + - common Pulumi transformations, enums and IP address ranges + - components + - composite + - a logical group of existing Pulumi components that is used in several places + - dynamic + - a custom component to implement some functionality that is not natively supported + - wrapped + - thin wrappers around Pulumi resources to expose additional methods/attributes + - stacks + - definitions of the `shm` and `sre` stacks +- provisioning + - all configuration options that is currently done outside Pulumi + - eg. Initialise the Guacamole database, reboot some VMs, create security groups on domain controller + - in the future this could be replaced by better orchestration options (eg. Ansible) or moved into Pulumi +- resources + - configuration files and templates used by Pulumi (e.g. cloud-init configs, Caddyfiles etc.) +- utility + - Useful classes: logging, file reading, types diff --git a/data_safe_haven/config/backend_settings.py b/data_safe_haven/config/backend_settings.py index ebc20e455e..61816839cf 100644 --- a/data_safe_haven/config/backend_settings.py +++ b/data_safe_haven/config/backend_settings.py @@ -88,7 +88,7 @@ def name(self) -> str: if not self._name: msg = ( "Data Safe Haven deployment name not provided:" - " use '[bright_cyan]--deployment-name[/]' / '[green]-d[/]' to do so." + " use '[bright_cyan]--name[/]' / '[green]-n[/]' to do so." ) raise DataSafeHavenParameterError(msg) return self._name diff --git a/data_safe_haven/config/config.py b/data_safe_haven/config/config.py index 06e3398c52..1acc30ec90 100644 --- a/data_safe_haven/config/config.py +++ b/data_safe_haven/config/config.py @@ -12,7 +12,11 @@ from yaml.parser import ParserError from data_safe_haven import __version__ -from data_safe_haven.exceptions import DataSafeHavenAzureError, DataSafeHavenConfigError +from data_safe_haven.exceptions import ( + DataSafeHavenAzureError, + DataSafeHavenConfigError, + DataSafeHavenParameterError, +) from data_safe_haven.external import AzureApi from data_safe_haven.functions import ( alphanumeric, @@ -352,7 +356,12 @@ def __init__(self) -> None: self.sres: dict[str, ConfigSectionSRE] = defaultdict(ConfigSectionSRE) # Read backend settings settings = BackendSettings() - self.name = settings.name + # Check if backend exists and was loaded + try: + self.name = settings.name + except DataSafeHavenParameterError as exc: + msg = "Data Safe Haven has not been initialised: run '[bright_cyan]dsh init[/]' before continuing." + raise DataSafeHavenConfigError(msg) from exc self.subscription_name = settings.subscription_name self.azure.location = settings.location self.azure.admin_group_id = settings.admin_group_id diff --git a/data_safe_haven/external/api/azure_api.py b/data_safe_haven/external/api/azure_api.py index 552bc35713..0ce8044bce 100644 --- a/data_safe_haven/external/api/azure_api.py +++ b/data_safe_haven/external/api/azure_api.py @@ -1084,6 +1084,42 @@ def run_remote_script( msg = f"Failed to run command on '{vm_name}'.\n{exc}" raise DataSafeHavenAzureError(msg) from exc + def run_remote_script_waiting( + self, + resource_group_name: str, + script: str, + script_parameters: dict[str, str], + vm_name: str, + ) -> str: + """Run a script on a remote virtual machine waiting for other scripts to complete + + Returns: + str: The script output + + Raises: + DataSafeHavenAzureError if running the script failed + """ + while True: + try: + script_output = self.run_remote_script( + resource_group_name=resource_group_name, + script=script, + script_parameters=script_parameters, + vm_name=vm_name, + ) + break + except DataSafeHavenAzureError as exc: + if all( + reason not in str(exc) + for reason in ( + "The request failed due to conflict with a concurrent request", + "Run command extension execution is in progress", + ) + ): + raise + time.sleep(5) + return script_output + def set_blob_container_acl( self, container_name: str, diff --git a/data_safe_haven/external/api/graph_api.py b/data_safe_haven/external/api/graph_api.py index a329b50437..1b1a00b677 100644 --- a/data_safe_haven/external/api/graph_api.py +++ b/data_safe_haven/external/api/graph_api.py @@ -361,7 +361,9 @@ def create_token_administrator(self) -> str: result = None try: # Load local token cache - local_token_cache = LocalTokenCache(pathlib.Path.home() / ".msal_cache") + local_token_cache = LocalTokenCache( + pathlib.Path.home() / f".msal_cache_{self.tenant_id}" + ) # Use the Powershell application by default as this should be pre-installed app = PublicClientApplication( authority=f"https://login.microsoftonline.com/{self.tenant_id}", diff --git a/data_safe_haven/functions/strings.py b/data_safe_haven/functions/strings.py index a3dac5e106..27089eeaed 100644 --- a/data_safe_haven/functions/strings.py +++ b/data_safe_haven/functions/strings.py @@ -90,7 +90,7 @@ def seeded_uuid(seed: str) -> uuid.UUID: def sha256hash(input_string: str) -> str: """Return the SHA256 hash of a string as a string.""" - return hashlib.sha256(str.encode(input_string, encoding="utf-8")).hexdigest() + return hashlib.sha256(input_string.encode("utf-8")).hexdigest() def truncate_tokens(tokens: Sequence[str], max_length: int) -> list[str]: diff --git a/data_safe_haven/infrastructure/components/__init__.py b/data_safe_haven/infrastructure/components/__init__.py index ee872fabe0..6fcb8d3f9b 100644 --- a/data_safe_haven/infrastructure/components/__init__.py +++ b/data_safe_haven/infrastructure/components/__init__.py @@ -20,6 +20,8 @@ CompiledDscProps, FileShareFile, FileShareFileProps, + FileUpload, + FileUploadProps, RemoteScript, RemoteScriptProps, SSLCertificate, @@ -41,6 +43,8 @@ "CompiledDscProps", "FileShareFile", "FileShareFileProps", + "FileUpload", + "FileUploadProps", "LinuxVMComponentProps", "LocalDnsRecordComponent", "LocalDnsRecordProps", diff --git a/data_safe_haven/infrastructure/components/dynamic/__init__.py b/data_safe_haven/infrastructure/components/dynamic/__init__.py index 2fe0f8decb..4fdfb12dfc 100644 --- a/data_safe_haven/infrastructure/components/dynamic/__init__.py +++ b/data_safe_haven/infrastructure/components/dynamic/__init__.py @@ -2,7 +2,8 @@ from .blob_container_acl import BlobContainerAcl, BlobContainerAclProps from .compiled_dsc import CompiledDsc, CompiledDscProps from .file_share_file import FileShareFile, FileShareFileProps -from .remote_powershell import RemoteScript, RemoteScriptProps +from .file_upload import FileUpload, FileUploadProps +from .remote_script import RemoteScript, RemoteScriptProps from .ssl_certificate import SSLCertificate, SSLCertificateProps __all__ = [ @@ -14,6 +15,8 @@ "CompiledDscProps", "FileShareFile", "FileShareFileProps", + "FileUpload", + "FileUploadProps", "RemoteScript", "RemoteScriptProps", "SSLCertificate", diff --git a/data_safe_haven/infrastructure/components/dynamic/file_upload.py b/data_safe_haven/infrastructure/components/dynamic/file_upload.py new file mode 100644 index 0000000000..4f1f259c47 --- /dev/null +++ b/data_safe_haven/infrastructure/components/dynamic/file_upload.py @@ -0,0 +1,144 @@ +"""Pulumi dynamic component for running remote scripts on an Azure VM.""" +from typing import Any + +from pulumi import Input, Output, ResourceOptions +from pulumi.dynamic import CreateResult, DiffResult, Resource, UpdateResult + +from data_safe_haven.exceptions import DataSafeHavenAzureError +from data_safe_haven.external import AzureApi +from data_safe_haven.functions import b64encode + +from .dsh_resource_provider import DshResourceProvider + + +class FileUploadProps: + """Props for the FileUpload class""" + + def __init__( + self, + file_contents: Input[str], + file_hash: Input[str], + file_permissions: Input[str], + file_target: Input[str], + subscription_name: Input[str], + vm_name: Input[str], + vm_resource_group_name: Input[str], + force_refresh: Input[bool] | None = None, + ) -> None: + self.file_contents = file_contents + self.file_hash = file_hash + self.file_target = file_target + self.file_permissions = file_permissions + self.force_refresh = Output.from_input(force_refresh).apply( + lambda force: force if force else False + ) + self.subscription_name = subscription_name + self.vm_name = vm_name + self.vm_resource_group_name = vm_resource_group_name + + +class FileUploadProvider(DshResourceProvider): + def create(self, props: dict[str, Any]) -> CreateResult: + """Run a remote script to create a file on a VM""" + outs = dict(**props) + azure_api = AzureApi(props["subscription_name"], disable_logging=True) + script_contents = f""" + target_dir=$(dirname "$target"); + mkdir -p $target_dir 2> /dev/null; + echo $contents_b64 | base64 --decode > $target; + chmod {props['file_permissions']} $target; + if [ -f "$target" ]; then + echo "Wrote file to $target"; + else + echo "Failed to write file to $target"; + fi + """ + script_parameters = { + "contents_b64": b64encode(props["file_contents"]), + "target": props["file_target"], + } + # Run remote script + script_output = azure_api.run_remote_script_waiting( + props["vm_resource_group_name"], + script_contents, + script_parameters, + props["vm_name"], + ) + outs["script_output"] = "\n".join( + [ + line.strip() + for line in script_output.replace("Enable succeeded:", "").split("\n") + if line + ] + ) + if "Failed to write" in outs["script_output"]: + raise DataSafeHavenAzureError(outs["script_output"]) + return CreateResult( + f"FileUpload-{props['file_hash']}", + outs=outs, + ) + + def delete(self, id_: str, props: dict[str, Any]) -> None: + """Delete the remote file from the VM""" + # Use `id` as a no-op to avoid ARG002 while maintaining function signature + id(id_) + azure_api = AzureApi(props["subscription_name"], disable_logging=True) + script_contents = """ + rm -f "$target"; + echo "Removed file at $target"; + """ + script_parameters = { + "target": props["file_target"], + } + # Run remote script + azure_api.run_remote_script_waiting( + props["vm_resource_group_name"], + script_contents, + script_parameters, + props["vm_name"], + ) + + def diff( + self, + id_: str, + old_props: dict[str, Any], + new_props: dict[str, Any], + ) -> DiffResult: + """Calculate diff between old and new state""" + # Use `id` as a no-op to avoid ARG002 while maintaining function signature + id(id_) + if new_props["force_refresh"]: + return DiffResult( + changes=True, + replaces=list(new_props.keys()), + stables=[], + delete_before_replace=False, + ) + return self.partial_diff(old_props, new_props, []) + + def update( + self, + id_: str, + old_props: dict[str, Any], + new_props: dict[str, Any], + ) -> UpdateResult: + """Updating is creating without the need to delete.""" + # Use `id` as a no-op to avoid ARG002 while maintaining function signature + id((id_, old_props)) + updated = self.create(new_props) + return UpdateResult(outs=updated.outs) + + +class FileUpload(Resource): + script_output: Output[str] + _resource_type_name = "dsh:common:FileUpload" # set resource type + + def __init__( + self, + name: str, + props: FileUploadProps, + opts: ResourceOptions | None = None, + ): + super().__init__( + FileUploadProvider(), name, {"script_output": None, **vars(props)}, opts + ) diff --git a/data_safe_haven/infrastructure/components/dynamic/remote_powershell.py b/data_safe_haven/infrastructure/components/dynamic/remote_script.py similarity index 100% rename from data_safe_haven/infrastructure/components/dynamic/remote_powershell.py rename to data_safe_haven/infrastructure/components/dynamic/remote_script.py diff --git a/data_safe_haven/infrastructure/stack_manager.py b/data_safe_haven/infrastructure/stack_manager.py index 0023ceff73..0acc55b7e6 100644 --- a/data_safe_haven/infrastructure/stack_manager.py +++ b/data_safe_haven/infrastructure/stack_manager.py @@ -39,6 +39,8 @@ def __init__( self.work_dir = config.work_directory / "pulumi" / self.program.short_name self.work_dir.mkdir(parents=True, exist_ok=True) self.login() # Log in to the Pulumi backend + self.initialise_workdir() + self.install_plugins() @property def local_stack_path(self) -> pathlib.Path: @@ -137,8 +139,6 @@ def copy_secret(self, name: str, other_stack: "StackManager") -> None: def deploy(self, *, force: bool = False) -> None: """Deploy the infrastructure with Pulumi.""" try: - self.initialise_workdir() - self.install_plugins() self.apply_config_options() if force: self.cancel() @@ -357,8 +357,6 @@ def set_config(self, name: str, value: str, *, secret: bool) -> None: def teardown(self) -> None: """Teardown the infrastructure deployed with Pulumi.""" try: - self.initialise_workdir() - self.install_plugins() self.refresh() self.destroy() self.remove_workdir() diff --git a/data_safe_haven/infrastructure/stacks/declarative_sre.py b/data_safe_haven/infrastructure/stacks/declarative_sre.py index 439da4b8ca..907ebfe667 100644 --- a/data_safe_haven/infrastructure/stacks/declarative_sre.py +++ b/data_safe_haven/infrastructure/stacks/declarative_sre.py @@ -253,6 +253,7 @@ def run(self) -> None: storage_account_data_private_user_name=data.storage_account_data_private_user_name, storage_account_data_private_sensitive_name=data.storage_account_data_private_sensitive_name, subnet_workspaces=networking.subnet_workspaces, + subscription_name=self.cfg.subscription_name, virtual_network_resource_group=networking.resource_group, virtual_network=networking.virtual_network, vm_details=list(enumerate(self.cfg.sres[self.sre_name].workspace_skus)), diff --git a/data_safe_haven/infrastructure/stacks/sre/workspaces.py b/data_safe_haven/infrastructure/stacks/sre/workspaces.py index 2821b6f669..fdf3d46c4d 100644 --- a/data_safe_haven/infrastructure/stacks/sre/workspaces.py +++ b/data_safe_haven/infrastructure/stacks/sre/workspaces.py @@ -1,3 +1,4 @@ +import pathlib from collections.abc import Mapping from typing import Any @@ -14,10 +15,13 @@ get_name_from_vnet, ) from data_safe_haven.infrastructure.components import ( + FileUpload, + FileUploadProps, LinuxVMComponentProps, VMComponent, ) from data_safe_haven.resources import resources_path +from data_safe_haven.utility import FileReader class SREWorkspacesProps: @@ -43,6 +47,7 @@ def __init__( storage_account_data_private_user_name: Input[str], storage_account_data_private_sensitive_name: Input[str], subnet_workspaces: Input[network.GetSubnetResult], + subscription_name: Input[str], virtual_network_resource_group: Input[resources.ResourceGroup], virtual_network: Input[network.VirtualNetwork], vm_details: list[tuple[int, str]], # this must *not* be passed as an Input[T] @@ -69,6 +74,7 @@ def __init__( self.storage_account_data_private_sensitive_name = ( storage_account_data_private_sensitive_name ) + self.subscription_name = subscription_name self.virtual_network_name = Output.from_input(virtual_network).apply( get_name_from_vnet ) @@ -161,7 +167,7 @@ def __init__( ] # Get details for each deployed VM - vm_outputs = [ + vm_outputs: list[dict[str, Any]] = [ { "ip_address": vm.ip_address_private, "name": vm.vm_name, @@ -170,6 +176,36 @@ def __init__( for vm in vms ] + # Upload smoke tests + mustache_values = { + "check_uninstallable_packages": "0", + } + file_uploads = [ + (FileReader(resources_path / "workspace" / "run_all_tests.bats"), "0444") + ] + for test_file in pathlib.Path(resources_path / "workspace").glob("test*"): + file_uploads.append((FileReader(test_file), "0444")) + for vm, vm_output in zip(vms, vm_outputs, strict=True): + outputs: dict[str, Output[str]] = {} + for file_upload, file_permissions in file_uploads: + file_smoke_test = FileUpload( + replace_separators(f"{self._name}_file_{file_upload.name}", "_"), + FileUploadProps( + file_contents=file_upload.file_contents( + mustache_values=mustache_values + ), + file_hash=file_upload.sha256(), + file_permissions=file_permissions, + file_target=f"/opt/tests/{file_upload.name}", + subscription_name=props.subscription_name, + vm_name=vm.vm_name, + vm_resource_group_name=resource_group.name, + ), + opts=child_opts, + ) + outputs[file_upload.name] = file_smoke_test.script_output + vm_output["file_uploads"] = outputs + # Register outputs self.resource_group = resource_group diff --git a/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist b/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist index d65ef196ea..9624ec7060 100644 --- a/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist +++ b/data_safe_haven/resources/software_repositories/allowlists/cran.allowlist @@ -1,4 +1,5 @@ DBI +MASS RPostgres Rcpp bit diff --git a/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist b/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist index 3ab3c07dfe..704937893f 100644 --- a/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist +++ b/data_safe_haven/resources/software_repositories/allowlists/pypi.allowlist @@ -15,6 +15,7 @@ pyodbc pyparsing python-dateutil pytz +scikit-learn six typing-extensions tzdata diff --git a/data_safe_haven/resources/workspace/run_all_tests.bats b/data_safe_haven/resources/workspace/run_all_tests.bats new file mode 100644 index 0000000000..800a55cd3d --- /dev/null +++ b/data_safe_haven/resources/workspace/run_all_tests.bats @@ -0,0 +1,128 @@ +#! /usr/bin/env bats + + +# Helper functions +# ---------------- +initialise_python_environment() { + ENV_PATH="${HOME}/.local/bats-python-environment" + rm -rf "$ENV_PATH" + python -m venv "$ENV_PATH" + source "${ENV_PATH}/bin/activate" + pip install --upgrade pip --quiet +} + +initialise_r_environment() { + ENV_PATH="${HOME}/.local/bats-r-environment" + rm -rf "$ENV_PATH" + mkdir -p "$ENV_PATH" +} + +install_r_package() { + PACKAGE_NAME="$1" + ENV_PATH="${HOME}/.local/bats-r-environment" + Rscript -e "install.packages('$PACKAGE_NAME', lib='$ENV_PATH');" +} + +install_r_package_version() { + PACKAGE_NAME="$1" + PACKAGE_VERSION="$2" + ENV_PATH="${HOME}/.local/bats-r-environment" + Rscript -e "install.packages('remotes', lib='$ENV_PATH');" + Rscript -e "library('remotes', lib='$ENV_PATH'); remotes::install_version(package='$PACKAGE_NAME', version='$PACKAGE_VERSION', lib='$ENV_PATH');" +} + +check_db_credentials() { + db_credentials="${HOME}/.local/db.dsh" + if [ -f "$db_credentials" ]; then + return 0 + fi + return 1 +} + + +# Mounted drives +# -------------- +@test "Mounted drives (/data)" { + run bash test_mounted_drives.sh -d data + [ "$status" -eq 0 ] +} +@test "Mounted drives (/home)" { + run bash test_mounted_drives.sh -d home + [ "$status" -eq 0 ] +} +@test "Mounted drives (/output)" { + run bash test_mounted_drives.sh -d output + [ "$status" -eq 0 ] +} +@test "Mounted drives (/shared)" { + run bash test_mounted_drives.sh -d shared + [ "$status" -eq 0 ] +} + + +# Package repositories +# -------------------- +@test "Python package repository" { + initialise_python_environment + run bash test_repository_python.sh 2>&1 + [ "$status" -eq 0 ] +} +@test "R package repository" { + initialise_r_environment + run bash test_repository_R.sh + [ "$status" -eq 0 ] +} + + +# Language functionality +# ---------------------- +@test "Python functionality" { + initialise_python_environment + pip install numpy pandas scikit-learn --quiet + run python test_functionality_python.py 2>&1 + [ "$status" -eq 0 ] +} +@test "R functionality" { + initialise_r_environment + install_r_package_version "MASS" "7.3-52" + run Rscript test_functionality_R.R + [ "$status" -eq 0 ] +} + + +# Databases +# --------- +# Test MS SQL database +@test "MS SQL database (Python)" { + check_db_credentials || skip "No database credentials available" + initialise_python_environment + pip install pandas psycopg pymssql --quiet + run bash test_databases.sh -d mssql -l python + [ "$status" -eq 0 ] +} +@test "MS SQL database (R)" { + check_db_credentials || skip "No database credentials available" + initialise_r_environment + install_r_package "DBI" + install_r_package "odbc" + install_r_package "RPostgres" + run bash test_databases.sh -d mssql -l R + [ "$status" -eq 0 ] +} +# Test Postgres database +@test "Postgres database (Python)" { + check_db_credentials || skip "No database credentials available" + initialise_python_environment + pip install pandas psycopg pymssql --quiet + run bash test_databases.sh -d postgresql -l python + [ "$status" -eq 0 ] +} +@test "Postgres database (R)" { + check_db_credentials || skip "No database credentials available" + initialise_r_environment + install_r_package "DBI" + install_r_package "odbc" + install_r_package "RPostgres" + run bash test_databases.sh -d postgresql -l R + [ "$status" -eq 0 ] +} diff --git a/data_safe_haven/resources/workspace/test_databases.sh b/data_safe_haven/resources/workspace/test_databases.sh new file mode 100644 index 0000000000..69fd7a456c --- /dev/null +++ b/data_safe_haven/resources/workspace/test_databases.sh @@ -0,0 +1,51 @@ +#! /bin/bash +db_type="" +language="" +while getopts d:l: flag; do + case "${flag}" in + d) db_type=${OPTARG} ;; + l) language=${OPTARG} ;; + *) + echo "Invalid option ${OPTARG}" + exit 1 + ;; + esac +done + +db_credentials="${HOME}/.local/db.dsh" +if [ -f "$db_credentials" ]; then + username="databaseadmin" + password="$(cat "$db_credentials")" +else + echo "Credentials file ($db_credentials) not found." + exit 1 +fi + +sre_fqdn="$(grep trusted /etc/pip.conf | cut -d "." -f 2-99)" +sre_prefix="$(hostname | cut -d "-" -f 1-4)" +if [ "$db_type" == "mssql" ]; then + db_name="master" + port="1433" + server_name="mssql.${sre_fqdn}" + hostname="${sre_prefix}-db-server-mssql" +elif [ "$db_type" == "postgresql" ]; then + db_name="postgres" + port="5432" + server_name="postgresql.${sre_fqdn}" + hostname="${sre_prefix}-db-server-postgresql" +else + echo "Did not recognise database type '$db_type'" + exit 1 +fi + +if [ "$port" == "" ]; then + echo "Database type '$db_type' is not part of this SRE" + exit 1 +else + script_path=$(dirname "$(readlink -f "$0")") + if [ "$language" == "python" ]; then + python "${script_path}"/test_databases_python.py --db-type "$db_type" --db-name "$db_name" --port "$port" --server-name "$server_name" --hostname "$hostname" --username "$username" --password "$password" || exit 1 + elif [ "$language" == "R" ]; then + Rscript "${script_path}"/test_databases_R.R "$db_type" "$db_name" "$port" "$server_name" "$hostname" "$username" "$password" || exit 1 + fi +fi diff --git a/data_safe_haven/resources/workspace/test_databases_R.R b/data_safe_haven/resources/workspace/test_databases_R.R new file mode 100644 index 0000000000..a261f21532 --- /dev/null +++ b/data_safe_haven/resources/workspace/test_databases_R.R @@ -0,0 +1,51 @@ +#!/usr/bin/env Rscript +library(DBI, lib.loc='~/.local/bats-r-environment') +library(odbc, lib.loc='~/.local/bats-r-environment') +library(RPostgres, lib.loc='~/.local/bats-r-environment') + +# Parse command line arguments +args = commandArgs(trailingOnly=TRUE) +if (length(args)!=7) { + stop("Exactly seven arguments are required: db_type, db_name, port, server_name, hostname, username and password") +} +db_type = args[1] +db_name = args[2] +port = args[3] +server_name = args[4] +hostname = args[5] +username = args[6] +password = args[7] + +# Connect to the database +print(paste("Attempting to connect to '", db_name, "' on '", server_name, "' via port '", port, sep="")) +if (db_type == "mssql") { + cnxn <- DBI::dbConnect( + odbc::odbc(), + Driver = "ODBC Driver 17 for SQL Server", + Server = paste(server_name, port, sep=","), + Database = db_name, + # Trusted_Connection = "yes", + UID = paste(username, "@", hostname, sep=""), + PWD = password + ) +} else if (db_type == "postgresql") { + cnxn <- DBI::dbConnect( + RPostgres::Postgres(), + host = server_name, + port = port, + dbname = db_name, + user = paste(username, "@", hostname, sep=""), + password = password + ) +} else { + stop(paste("Database type '", db_type, "' was not recognised", sep="")) +} + +# Run a query and save the output into a dataframe +df <- dbGetQuery(cnxn, "SELECT * FROM information_schema.tables;") +if (dim(df)[1] > 0) { + print(head(df, 5)) + print("All database tests passed") +} else { + stop(paste("Reading from database '", db_name, "' failed", sep="")) +} diff --git a/data_safe_haven/resources/workspace/test_databases_python.py b/data_safe_haven/resources/workspace/test_databases_python.py new file mode 100644 index 0000000000..ab0f01a3fe --- /dev/null +++ b/data_safe_haven/resources/workspace/test_databases_python.py @@ -0,0 +1,66 @@ +#! /usr/bin/env python +import argparse + +import pandas as pd +import psycopg +import pymssql + + +def test_database( + server_name: str, + hostname: str, + port: int, + db_type: str, + db_name: str, + username: str, + password: str, +) -> None: + msg = f"Attempting to connect to '{db_name}' on '{server_name}' via port {port}" + print(msg) # noqa: T201 + username_full = f"{username}@{hostname}" + cnxn = None + if db_type == "mssql": + cnxn = pymssql.connect( + server=server_name, user=username_full, password=password, database=db_name + ) + elif db_type == "postgresql": + connection_string = f"host={server_name} port={port} dbname={db_name} user={username_full} password={password}" + cnxn = psycopg.connect(connection_string) + else: + msg = f"Database type '{db_type}' was not recognised" + raise ValueError(msg) + df = pd.read_sql("SELECT * FROM information_schema.tables;", cnxn) + if df.size: + print(df.head(5)) # noqa: T201 + print("All database tests passed") # noqa: T201 + else: + msg = f"Reading from database '{db_name}' failed." + raise ValueError(msg) + + +# Parse command line arguments +parser = argparse.ArgumentParser() +parser.add_argument( + "--db-type", + type=str, + choices=["mssql", "postgresql"], + help="Which database type to use", +) +parser.add_argument("--db-name", type=str, help="Which database to connect to") +parser.add_argument("--port", type=str, help="Which port to connect to") +parser.add_argument("--server-name", type=str, help="Which server to connect to") +parser.add_argument("--username", type=str, help="Database username") +parser.add_argument("--hostname", type=str, help="Azure hostname of the server") +parser.add_argument("--password", type=str, help="Database user password") +args = parser.parse_args() + +# Run database test +test_database( + args.server_name, + args.hostname, + args.port, + args.db_type, + args.db_name, + args.username, + args.password, +) diff --git a/data_safe_haven/resources/workspace/test_functionality_R.R b/data_safe_haven/resources/workspace/test_functionality_R.R new file mode 100644 index 0000000000..94c351e7c3 --- /dev/null +++ b/data_safe_haven/resources/workspace/test_functionality_R.R @@ -0,0 +1,39 @@ +# Test logistic regression using R +library('MASS', lib.loc='~/.local/bats-r-environment') +library('stats') + +gen_data <- function(n = 100, p = 3) { + set.seed(1) + weights <- stats::rgamma(n = n, shape = rep(1, n), rate = rep(1, n)) + y <- stats::rbinom(n = n, size = 1, prob = 0.5) + theta <- stats::rnorm(n = p, mean = 0, sd = 1) + means <- colMeans(as.matrix(y) %*% theta) + x <- MASS::mvrnorm(n = n, means, diag(1, p, p)) + return(list(x = x, y = y, weights = weights, theta = theta)) +} + +run_logistic_regression <- function(data) { + fit <- stats::glm.fit(x = data$x, + y = data$y, + weights = data$weights, + family = stats::quasibinomial(link = "logit")) + return(fit$coefficients) +} + +data <- gen_data() +theta <- run_logistic_regression(data) +print("Logistic regression ran OK") + + +# Test clustering of random data using R +num_clusters <- 5 +N <- 10 +set.seed(0, kind = "Mersenne-Twister") +cluster_means <- runif(num_clusters, 0, 10) +means_selector <- as.integer(runif(N, 1, num_clusters + 1)) +data_means <- cluster_means[means_selector] +data <- rnorm(n = N, mean = data_means, sd = 0.5) +hc <- hclust(dist(data)) +print("Clustering ran OK") + +print("All functionality tests passed") diff --git a/data_safe_haven/resources/workspace/test_functionality_python.py b/data_safe_haven/resources/workspace/test_functionality_python.py new file mode 100644 index 0000000000..855e5e5f15 --- /dev/null +++ b/data_safe_haven/resources/workspace/test_functionality_python.py @@ -0,0 +1,37 @@ +"""Test logistic regression using python""" +import numpy as np +import pandas as pd +from sklearn.linear_model import LogisticRegression + + +def gen_data(n_samples: int, n_points: int) -> pd.DataFrame: + """Generate data for fitting""" + target = np.random.binomial(n=1, p=0.5, size=(n_samples, 1)) + theta = np.random.normal(loc=0.0, scale=1.0, size=(1, n_points)) + means = np.mean(np.multiply(target, theta), axis=0) + values = np.random.multivariate_normal( + means, np.diag([1] * n_points), size=n_samples + ).T + data = {f"x{n}": values[n] for n in range(n_points)} + data["y"] = target.reshape((n_samples,)) + data["weights"] = np.random.gamma(shape=1, scale=1.0, size=n_samples) + return pd.DataFrame(data=data) + + +def main() -> None: + """Logistic regression""" + data = gen_data(100, 3) + input_data = data.iloc[:, :-2] + output_data = data["y"] + weights = data["weights"] + + logit = LogisticRegression(solver="liblinear") + logit.fit(input_data, output_data, sample_weight=weights) + logit.score(input_data, output_data, sample_weight=weights) + + print("Logistic model ran OK") # noqa: T201 + print("All functionality tests passed") # noqa: T201 + + +if __name__ == "__main__": + main() diff --git a/data_safe_haven/resources/workspace/test_mounted_drives.sh b/data_safe_haven/resources/workspace/test_mounted_drives.sh new file mode 100644 index 0000000000..a1812934b9 --- /dev/null +++ b/data_safe_haven/resources/workspace/test_mounted_drives.sh @@ -0,0 +1,66 @@ +#! /bin/bash +while getopts d: flag +do + case "${flag}" in + d) directory=${OPTARG};; + *) + echo "Usage: $0 -d [directory]" + exit 1 + esac +done + +nfailed=0 +if [[ "$directory" = "home" ]]; then directory_path=$(echo ~); else directory_path="/${directory}"; fi +testfile="$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1)" + +# Check that directory exists +if [ "$(ls "${directory_path}" 2>&1 1>/dev/null)" ]; then + echo "Could not find mount '${directory_path}'" + nfailed=$((nfailed + 1)) +fi + +# Test operations +CAN_CREATE="$([[ "$(touch "${directory_path}/${testfile}" 2>&1 1>/dev/null)" = "" ]] && echo '1' || echo '0')" +CAN_WRITE="$([[ -w "${directory_path}/${testfile}" ]] && echo '1' || echo '0')" +CAN_DELETE="$([[ "$(touch "${directory_path}/${testfile}" 2>&1 1>/dev/null && rm "${directory_path}/${testfile}" 2>&1)" ]] && echo '0' || echo '1')" + +# Check that permissions are as expected for each directory +case "$directory" in + data) + if [ "$CAN_CREATE" = 1 ]; then echo "Able to create files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_WRITE" = 1 ]; then echo "Able to write files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_DELETE" = 1 ]; then echo "Able to delete files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + ;; + + home) + if [ "$CAN_CREATE" = 0 ]; then echo "Unable to create files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_WRITE" = 0 ]; then echo "Unable to write files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_DELETE" = 0 ]; then echo "Unable to delete files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + ;; + + output) + if [ "$CAN_CREATE" = 0 ]; then echo "Unable to create files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_WRITE" = 0 ]; then echo "Unable to write files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_DELETE" = 0 ]; then echo "Unable to delete files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + ;; + + shared) + if [ "$CAN_CREATE" = 0 ]; then echo "Unable to create files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_WRITE" = 0 ]; then echo "Unable to write files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + if [ "$CAN_DELETE" = 0 ]; then echo "Unable to delete files in ${directory_path}!"; nfailed=$((nfailed + 1)); fi + ;; + + *) + echo "Usage: $0 -d [directory]" + exit 1 +esac + +# Cleanup and print output +rm -f "${directory_path}/${testfile}" 2> /dev/null +if [ $nfailed = 0 ]; then + echo "All tests passed for '${directory_path}'" + exit 0 +else + echo "$nfailed tests failed for '${directory_path}'!" + exit $nfailed +fi diff --git a/data_safe_haven/resources/workspace/test_repository_R.mustache.sh b/data_safe_haven/resources/workspace/test_repository_R.mustache.sh new file mode 100644 index 0000000000..03568b1e62 --- /dev/null +++ b/data_safe_haven/resources/workspace/test_repository_R.mustache.sh @@ -0,0 +1,49 @@ +#! /bin/bash +# We need to test packages that are: +# - *not* pre-installed +# - on the tier-3 list (so we can test all tiers) +# - alphabetically early and late (so we can test the progress of the mirror synchronisation) +packages=("askpass" "zeallot") +uninstallable_packages=("aws.s3") + +# Create a temporary library directory +TEST_INSTALL_PATH="${HOME}/.local/bats-r-environment" +# TEST_INSTALL_PATH="${HOME}/test-repository-R" +# rm -rf "$TEST_INSTALL_PATH" +# mkdir -p "$TEST_INSTALL_PATH" + +# Install sample packages to local user library +N_FAILURES=0 +for package in "${packages[@]}"; do + echo "Attempting to install ${package}..." + Rscript -e "options(warn=-1); install.packages('${package}', lib='${TEST_INSTALL_PATH}', quiet=TRUE)" + if (Rscript -e "library('${package}', lib.loc='${TEST_INSTALL_PATH}')"); then + echo "... $package installation succeeded" + else + echo "... $package installation failed" + N_FAILURES=$((N_FAILURES + 1)) + fi +done +# If requested, demonstrate that installation fails for packages *not* on the approved list +TEST_FAILURE="{{check_uninstallable_packages}}" +if [ $TEST_FAILURE -eq 1 ]; then + for package in "${uninstallable_packages[@]}"; do + echo "Attempting to install ${package}..." + Rscript -e "options(warn=-1); install.packages('${package}', lib='${TEST_INSTALL_PATH}', quiet=TRUE)" + if (Rscript -e "library('${package}', lib.loc='${TEST_INSTALL_PATH}')"); then + echo "... $package installation unexpectedly succeeded!" + N_FAILURES=$((N_FAILURES + 1)) + else + echo "... $package installation failed as expected" + fi + done +fi +rm -rf "$TEST_INSTALL_PATH" + +if [ $N_FAILURES -eq 0 ]; then + echo "All package installations behaved as expected" + exit 0 +else + echo "One or more package installations did not behave as expected!" + exit $N_FAILURES +fi diff --git a/data_safe_haven/resources/workspace/test_repository_python.mustache.sh b/data_safe_haven/resources/workspace/test_repository_python.mustache.sh new file mode 100644 index 0000000000..28e46a23e1 --- /dev/null +++ b/data_safe_haven/resources/workspace/test_repository_python.mustache.sh @@ -0,0 +1,42 @@ +#! /bin/bash + +# We need to test packages that are: +# - *not* pre-installed +# - on the allowlist (so we can test this is working) +# - alphabetically early and late (so we can test the progress of the mirror synchronisation) +installable_packages=("contourpy" "tzdata") +uninstallable_packages=("awscli") + +# Install sample packages to local user library +N_FAILURES=0 +for package in "${installable_packages[@]}"; do + echo "Attempting to install ${package}..." + if (pip install "$package" --quiet); then + echo "... $package installation succeeded" + else + echo "... $package installation failed" + N_FAILURES=$((N_FAILURES + 1)) + fi +done +# If requested, demonstrate that installation fails for packages *not* on the approved list +TEST_FAILURE="{{check_uninstallable_packages}}" +if [ $TEST_FAILURE -eq 1 ]; then + for package in "${uninstallable_packages[@]}"; do + echo "Attempting to install ${package}..." + if (pip install "$package" --quiet); then + echo "... $package installation unexpectedly succeeded!" + N_FAILURES=$((N_FAILURES + 1)) + else + echo "... $package installation failed as expected" + fi + done +fi +rm -rf "$TEST_INSTALL_PATH" + +if [ $N_FAILURES -eq 0 ]; then + echo "All package installations behaved as expected" + exit 0 +else + echo "One or more package installations did not behave as expected!" + exit $N_FAILURES +fi diff --git a/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml b/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml index 17471221de..c4216adb76 100644 --- a/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml +++ b/data_safe_haven/resources/workspace/workspace.cloud_init.mustache.yaml @@ -104,6 +104,8 @@ packages: - libpq-dev # interact with PostgreSQL databases - msodbcsql17 # interact with Microsoft SQL databases - unixodbc-dev # interact with Microsoft SQL databases + # Bash testing + - bats package_update: true package_upgrade: true diff --git a/deployment/CheckRequirements.ps1 b/deployment/CheckRequirements.ps1 index 740f081581..6fad3f01c4 100644 --- a/deployment/CheckRequirements.ps1 +++ b/deployment/CheckRequirements.ps1 @@ -8,7 +8,7 @@ param ( Import-Module $PSScriptRoot/common/Logging -Force -ErrorAction Stop # Requirements -$PowershellSupportedVersion = "7.3.2" +$PowershellSupportedVersion = "7.3.6" $ModuleVersionRequired = @{ "Az.Accounts" = @("ge", "2.11.1") "Az.Automation" = @("ge", "1.9.0") diff --git a/deployment/administration/SHM_Delete_Unassigned_Users.ps1 b/deployment/administration/SHM_Delete_Unassigned_Users.ps1 new file mode 100644 index 0000000000..c09cd6c7be --- /dev/null +++ b/deployment/administration/SHM_Delete_Unassigned_Users.ps1 @@ -0,0 +1,33 @@ +param( + [Parameter(Mandatory = $true, HelpMessage = "Enter SHM ID (e.g. use 'testa' for Turing Development Safe Haven A)")] + [string]$shmId, + [Parameter(Mandatory = $false, HelpMessage = "No-op mode which will not remove anything")] + [Switch]$dryRun +) + +Import-Module Az.Accounts -ErrorAction Stop +Import-Module $PSScriptRoot/../common/AzureCompute -Force -ErrorAction Stop +Import-Module $PSScriptRoot/../common/Configuration -Force -ErrorAction Stop +Import-Module $PSScriptRoot/../common/Logging -Force -ErrorAction Stop + +# Get config +# ------------------------------- +$config = Get-ShmConfig -shmId $shmId +$originalContext = Get-AzContext + +# Delete users not currently in a security group +# ---------------------------------------------- +$null = Set-AzContext -SubscriptionId $config.subscriptionName -ErrorAction Stop +$script = "remote/Delete_Unassigned_Users.ps1" + +# Passing a param to a remote script requires it to be a string +if ($dryRun.IsPresent) { + Add-LogMessage -Level Info "Listing users not assigned to any security group from $($config.dc.vmName)..." + $params = @{dryRun = "yes" } +} else { + Add-LogMessage -Level Info "Deleting users not assigned to any security group from $($config.dc.vmName)..." + $params = @{dryRun = "no" } +} +$result = Invoke-RemoteScript -Shell "PowerShell" -ScriptPath $script -VMName $config.dc.vmName -ResourceGroupName $config.dc.rg -Parameter $params + +$null = Set-AzContext -Context $originalContext -ErrorAction Stop \ No newline at end of file diff --git a/deployment/administration/remote/Delete_Unassigned_Users.ps1 b/deployment/administration/remote/Delete_Unassigned_Users.ps1 new file mode 100644 index 0000000000..c154d5a012 --- /dev/null +++ b/deployment/administration/remote/Delete_Unassigned_Users.ps1 @@ -0,0 +1,33 @@ +param( + [Parameter(Mandatory = $true, HelpMessage = "yes/no determines whether users should actually be deleted")] + [string]$dryRun +) + +# Extract list of users +$userOuPath = (Get-ADObject -Filter * | Where-Object { $_.Name -eq "Safe Haven Research Users" }).DistinguishedName +$users = Get-ADUser -Filter * -SearchBase "$userOuPath" -Properties * +foreach ($user in $users) { + $groupName = ($user | Select-Object -ExpandProperty MemberOf | ForEach-Object { (($_ -Split ",")[0] -Split "=")[1] }) -join "|" + if (!($groupName)) { + $name = $user.SamAccountName + if ($dryRun -eq "yes") { + Write-Output "User $name would be deleted by this action" + } else { + Write-Output "Deleting $name" + Remove-ADUser -Identity $name -Confirm:$false + } + } +} + +# Force sync with AzureAD. It will still take around 5 minutes for changes to propagate +if ($dryRun -eq "no") { + Write-Output "Synchronising locally Active Directory with Azure" + try { + Import-Module -Name "C:\Program Files\Microsoft Azure AD Sync\Bin\ADSync" -ErrorAction Stop + Start-ADSyncSyncCycle -PolicyType Delta + } catch [System.IO.FileNotFoundException] { + Write-Output "Skipping as Azure AD Sync is not installed" + } catch { + Write-Output "Unable to run Azure Active Directory synchronisation!" + } +} \ No newline at end of file diff --git a/deployment/common/Configuration.psm1 b/deployment/common/Configuration.psm1 index 9c21a51232..04989122fd 100644 --- a/deployment/common/Configuration.psm1 +++ b/deployment/common/Configuration.psm1 @@ -330,7 +330,7 @@ function Get-ShmConfig { ) # *-jobruntimedata-prod-su1.azure-automation.net linux = ( @("72.32.157.246", "87.238.57.227", "147.75.85.69", "217.196.149.55") + # apt.postgresql.org - @("91.189.91.38", "91.189.91.39", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com + @("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com $cloudFlareIpAddresses + # database.clamav.net, packages.gitlab.com and qgis.org use Cloudflare $cloudFrontIpAddresses + # packages.gitlab.com uses Cloudfront to host its Release file @("104.131.190.124") + # dbeaver.io @@ -729,7 +729,6 @@ function Get-SreConfig { } } - # Firewall config # --------------- $config.sre.firewall = [ordered]@{ @@ -928,28 +927,10 @@ function Get-SreConfig { $config.sre.remoteDesktop.networkRules.includeAzurePlatformDnsRule = ($config.sre.remoteDesktop.networkRules.outboundInternet -ne "Allow") - # CoCalc, CodiMD and Gitlab servers - # --------------------------------- + # CodiMD and Gitlab servers + # ------------------------- $config.sre.webapps = [ordered]@{ rg = "$($config.sre.rgPrefix)_WEBAPPS".ToUpper() - cocalc = [ordered]@{ - adminPasswordSecretName = "$($config.sre.shortName)-vm-admin-password-cocalc" - dockerVersion = "latest" - hostname = "COCALC" - vmSize = "Standard_D2s_v3" - ip = Get-NextAvailableIpInRange -IpRangeCidr $config.sre.network.vnet.subnets.webapps.cidr -Offset 7 - osVersion = "Ubuntu-latest" - disks = [ordered]@{ - data = [ordered]@{ - sizeGb = "512" - type = $config.sre.diskTypeDefault - } - os = [ordered]@{ - sizeGb = "32" - type = $config.sre.diskTypeDefault - } - } - } codimd = [ordered]@{ adminPasswordSecretName = "$($config.sre.shortName)-vm-admin-password-codimd" hostname = "CODIMD" diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml index 38b7e3cb87..9bcd26b283 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-cran.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml index 8c09932601..0b38914c3b 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-external-pypi.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: @@ -122,7 +122,7 @@ write_files: - path: "/etc/cron.d/pull-from-internet" permissions: "0644" content: | - # External update (rsync from CRAN) every 6 hours + # External update from PyPi every 6 hours 0 */6 * * * mirrordaemon ~mirrordaemon/pull_from_internet.sh - path: "/etc/cron.d/pull-then-push" @@ -313,7 +313,7 @@ runcmd: # Install bandersnatch with pip - echo ">=== Installing bandersnatch... ===<" - - pip3 install bandersnatch==4.2.0 + - pip3 install bandersnatch==4.2.0 packaging==21.3 - echo "Using bandersnatch from '$(which bandersnatch)'" # Initialise allowlist if appropriate diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml index 9f0a108f60..6c67f7d4dd 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-cran.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml index 2fad1ab1ee..3507b51dbd 100644 --- a/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml +++ b/deployment/safe_haven_management_environment/cloud_init/cloud-init-repository-mirror-internal-pypi.mustache.yaml @@ -9,7 +9,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml index 44e1bb4397..ce9b2a0630 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-1804.mustache.yaml @@ -114,11 +114,6 @@ write_files: content: | {{packages-python.yaml}} - - path: "/opt/build/pyenv/pyproject_template.toml" - permissions: "0400" - content: | - {{pyenv_pyproject_template.toml}} - - path: "/opt/build/rstudio.debinfo" permissions: "0400" content: | @@ -257,8 +252,6 @@ runcmd: # Get icons for webapps - echo "Downloading icons..." - mkdir -p /opt/icons - - wget https://raw.githubusercontent.com/sagemathinc/cocalc/master/src/packages/assets/cocalc-icon.svg -O /tmp/cocalc-icon.svg || die "Could not find CoCalc icon!" - - convert -density 983 -transparent white /tmp/cocalc-icon.svg /opt/icons/cocalc.png || die "Could not convert CoCalc icon!" # this should give a 1024x1024 png - wget https://raw.githubusercontent.com/hackmdio/codimd/develop/public/favicon.png -O /opt/icons/codimd.png || die "Could not find CodiMD icon!" - wget https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png -O /opt/icons/gitlab.png || die "Could not find GitLab icon!" diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml index 5605e07707..c1d5a53c2e 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2004.mustache.yaml @@ -114,11 +114,6 @@ write_files: content: | {{packages-python.yaml}} - - path: "/opt/build/pyenv/pyproject_template.toml" - permissions: "0400" - content: | - {{pyenv_pyproject_template.toml}} - - path: "/opt/build/rstudio.debinfo" permissions: "0400" content: | @@ -260,8 +255,6 @@ runcmd: # Get icons for webapps - echo "Downloading icons..." - mkdir -p /opt/icons - - wget https://raw.githubusercontent.com/sagemathinc/cocalc/master/src/packages/assets/cocalc-icon.svg -O /tmp/cocalc-icon.svg || die "Could not find CoCalc icon!" - - convert -density 983 -transparent white /tmp/cocalc-icon.svg /opt/icons/cocalc.png || die "Could not convert CoCalc icon!" # this should give a 1024x1024 png - wget https://raw.githubusercontent.com/hackmdio/codimd/develop/public/favicon.png -O /opt/icons/codimd.png || die "Could not find CodiMD icon!" - wget https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png -O /opt/icons/gitlab.png || die "Could not find GitLab icon!" diff --git a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml index 17527746cb..5cfcc671e7 100644 --- a/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml +++ b/deployment/secure_research_desktop/cloud_init/cloud-init-buildimage-ubuntu-2204.mustache.yaml @@ -118,11 +118,6 @@ write_files: content: | {{packages-python.yaml}} - - path: "/opt/build/pyenv/pyproject_template.toml" - permissions: "0400" - content: | - {{pyenv_pyproject_template.toml}} - - path: "/opt/build/rbase.debinfo" permissions: "0400" content: | @@ -266,8 +261,6 @@ runcmd: # Get icons for webapps - echo "Downloading icons..." - mkdir -p /opt/icons - - wget https://raw.githubusercontent.com/sagemathinc/cocalc/master/src/packages/assets/cocalc-icon.svg -O /tmp/cocalc-icon.svg || die "Could not find CoCalc icon!" - - convert -density 983 -transparent white /tmp/cocalc-icon.svg /opt/icons/cocalc.png || die "Could not convert CoCalc icon!" # this should give a 1024x1024 png - wget https://raw.githubusercontent.com/hackmdio/codimd/develop/public/favicon.png -O /opt/icons/codimd.png || die "Could not find CodiMD icon!" - wget https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png -O /opt/icons/gitlab.png || die "Could not find GitLab icon!" diff --git a/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh b/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh index 555c46c14f..56898d3915 100644 --- a/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh +++ b/deployment/secure_research_desktop/cloud_init/resources/pyenv_install.sh @@ -9,7 +9,6 @@ if [ $# -ne 1 ]; then fi PYTHON_VERSION=$1 PYENV_ROOT="$(pyenv root)" -PYPROJECT_TOML="/opt/build/python-${PYTHON_VERSION}-pyproject.toml" MONITORING_LOG="/opt/monitoring/python-${PYTHON_VERSION}-package-versions.log" REQUIREMENTS_TXT="/opt/build/python-${PYTHON_VERSION}-requirements.txt" REQUESTED_PACKAGE_LIST="/opt/build/packages/packages-python-${PYTHON_VERSION}.list" @@ -27,24 +26,19 @@ echo "Installed $(${EXE_PATH}/python --version)" # Install and upgrade installation prerequisites # ---------------------------------------------- echo "Installing and upgrading installation prerequisites for Python ${PYTHON_VERSION}..." -${EXE_PATH}/pip install --upgrade pip poetry +${EXE_PATH}/pip install --upgrade pip pip-tools setuptools -# Solve dependencies and install using poetry -# ------------------------------------------- -echo "Installing packages with poetry..." -${EXE_PATH}/poetry config virtualenvs.create false -${EXE_PATH}/poetry config virtualenvs.in-project true -rm poetry.lock pyproject.toml 2> /dev/null -sed -e "s/PYTHON_VERSION/$PYTHON_VERSION/" /opt/build/pyenv/pyproject_template.toml > $PYPROJECT_TOML -ln -s $PYPROJECT_TOML pyproject.toml -${EXE_PATH}/poetry add $(tr '\n' ' ' < $REQUIREMENTS_TXT) || exit 3 +# Solve dependencies and write package versions to monitoring log +# --------------------------------------------------------------- +echo "Determining package versions with pip-compile..." +${EXE_PATH}/pip-compile -o "$MONITORING_LOG" "$REQUIREMENTS_TXT" -# Write package versions to monitoring log -# ---------------------------------------- -${EXE_PATH}/poetry show > $MONITORING_LOG -${EXE_PATH}/poetry show --tree >> $MONITORING_LOG +# Install pinned packages using pip +# --------------------------------- +echo "Installing packages with pip..." +${EXE_PATH}/pip install -r "$MONITORING_LOG" # Run any post-install commands diff --git a/deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml b/deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml deleted file mode 100644 index 3f0998952a..0000000000 --- a/deployment/secure_research_desktop/cloud_init/resources/pyenv_pyproject_template.toml +++ /dev/null @@ -1,12 +0,0 @@ -[tool.poetry] -name = "Python PYTHON_VERSION" -version = "1.0.0" -description = "Python PYTHON_VERSION" -authors = ["ROOT "] - -[tool.poetry.dependencies] -python = "PYTHON_VERSION" - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" diff --git a/deployment/secure_research_desktop/packages/deb-azuredatastudio.version b/deployment/secure_research_desktop/packages/deb-azuredatastudio.version index a68729c515..c3b2138e29 100644 --- a/deployment/secure_research_desktop/packages/deb-azuredatastudio.version +++ b/deployment/secure_research_desktop/packages/deb-azuredatastudio.version @@ -1,4 +1,4 @@ -hash: c105a5286861ff1f312ab699d5810af7dc0e668917afb1f80c4eda3a58bb8d34 -version: 1.45.1 +hash: 6c75efb8596d25152bc44f6f59a4b96cd013219b0c87ae2365ca8f69fe29e206 +version: 1.46.0 debfile: azuredatastudio-linux-|VERSION|.deb -remote: https://sqlopsbuilds.azureedge.net/stable/88c21b1725a3e79440027bdb7b5a55fb036be0e2/|DEBFILE| +remote: https://sqlopsbuilds.azureedge.net/stable/39449bbe88a0bc4092c9b205cad10d0a556beffd/|DEBFILE| diff --git a/deployment/secure_research_desktop/packages/deb-rstudio-focal.version b/deployment/secure_research_desktop/packages/deb-rstudio-focal.version index fec4214082..6f11a52f50 100644 --- a/deployment/secure_research_desktop/packages/deb-rstudio-focal.version +++ b/deployment/secure_research_desktop/packages/deb-rstudio-focal.version @@ -1,4 +1,4 @@ -hash: 49e24a6956f9f12ffeded493f571cd39f3e6c89411fc60d3bb228661793320da -version: 2023.06.1-524 +hash: 981fcbb30d5dea283188fcef1a9cdf177bb51b83884a63fd3c9c224d1486b01e +version: 2023.06.2-561 debfile: rstudio-|VERSION|-amd64.deb remote: https://download1.rstudio.org/electron/focal/amd64/|DEBFILE| diff --git a/deployment/secure_research_desktop/packages/deb-rstudio-jammy.version b/deployment/secure_research_desktop/packages/deb-rstudio-jammy.version index 6c96a9ac17..69b2e3336a 100644 --- a/deployment/secure_research_desktop/packages/deb-rstudio-jammy.version +++ b/deployment/secure_research_desktop/packages/deb-rstudio-jammy.version @@ -1,4 +1,4 @@ -hash: c030ec8338f1c76b3ae27997ec4411a0af43b2367dedb3d48e95c319b5425698 -version: 2023.06.1-524 +hash: bb6b3c21510abb18fd6e697567d7ff3d4135bf7980cf25536753e9ceac60c82c +version: 2023.06.2-561 debfile: rstudio-|VERSION|-amd64.deb remote: https://download1.rstudio.org/electron/jammy/amd64/|DEBFILE| diff --git a/deployment/secure_research_desktop/packages/packages-python.yaml b/deployment/secure_research_desktop/packages/packages-python.yaml index 879788d373..d520b3d60e 100644 --- a/deployment/secure_research_desktop/packages/packages-python.yaml +++ b/deployment/secure_research_desktop/packages/packages-python.yaml @@ -39,9 +39,8 @@ packages: pathos: pg8000: Pillow: + pip-tools: plotly: - poetry: # also used by installation scripts - "all": [">1.0.0"] # increase solver flexibility prophet: psycopg2: pydot: diff --git a/deployment/secure_research_desktop/packages/packages-r-cran.list b/deployment/secure_research_desktop/packages/packages-r-cran.list index 0c716553bb..7cf7e374a6 100644 --- a/deployment/secure_research_desktop/packages/packages-r-cran.list +++ b/deployment/secure_research_desktop/packages/packages-r-cran.list @@ -1,3 +1,4 @@ +arrow BiocManager caret csv diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml deleted file mode 100644 index 43da1b27da..0000000000 --- a/deployment/secure_research_environment/cloud_init/cloud-init-cocalc.mustache.yaml +++ /dev/null @@ -1,278 +0,0 @@ -#cloud-config - -# Create files -write_files: - - path: "/etc/apt/apt.conf.d/00proxy" - permissions: "0444" - content: | - Acquire::http::Proxy "http://{{shm.monitoring.updateServers.linux.ip}}:8000"; - - - path: "/etc/audit/rules.d/audit.rules" - permissions: "0400" - content: | - {{audit.rules}} - - - path: "/etc/clamav/clamd.conf" - permissions: "0644" - append: true - content: | - {{clamd.conf}} - - - path: "/etc/cron.d/clamav-freshclam" - permissions: "0644" - content: | - # Run every day at a fixed time - {{shm.monitoring.updateServers.schedule.daily_definition_updates.minute}} {{shm.monitoring.updateServers.schedule.daily_definition_updates.hour}} * * * freshclam - - - path: "/etc/pip.conf" - permissions: "0444" - content: | - # Add the PyPI mirror to our global settings - [global] - index = {{sre.repositories.pypi.index}} - index-url = {{sre.repositories.pypi.indexUrl}} - trusted-host = {{sre.repositories.pypi.host}} - - - path: "/etc/R/Rprofile.site" - permissions: "0444" - content: | - ## Set Rprofile.site to the appropriate CRAN mirror - local({ - r <- getOption("repos") - r["CRAN"] <- "{{sre.repositories.cran.url}}" - options(repos = r) - }) - - - path: "/etc/systemd/system/clamav-clamonacc.service" - permissions: "0644" - content: | - {{clamav-clamonacc.service}} - - - path: "/etc/systemd/system/clamav-clamdscan.service" - permissions: "0644" - content: | - {{clamav-clamdscan.service}} - - - path: "/etc/systemd/system/clamav-clamdscan.timer" - permissions: "0644" - content: | - {{clamav-clamdscan.timer}} - - - path: "/etc/systemd/system/ingress.mount" - permissions: "0644" - content: | - [Unit] - Description=Mount unit for ingress volume (via blobfuse) - Requires=network-online.target - After=network-online.target - - [Mount] - What=/opt/mounts/ingress-mount.sh - Where=/ingress - Type=fuse - Options=_netdev - - [Install] - WantedBy=network-online.target - - - path: "/etc/systemd/system/ingress.automount" - permissions: "0644" - content: | - [Unit] - Description=Automount blobfuse - ConditionPathExists=/ingress - - [Automount] - Where=/ingress - TimeoutIdleSec=10 - - [Install] - WantedBy=multi-user.target - - - path: "/opt/cocalc/docker-compose.yaml" - permissions: "0400" - content: | - {{cocalc_docker_compose.mustache.yaml}} - - - path: "/opt/configuration/set_dns.sh" - permissions: "0500" - content: | - {{set_dns.mustache.sh}} - - - path: "/opt/mounts/ingress-credentials.secret" - permissions: "0400" - content: | - accountName {{sre.storage.persistentdata.account.name}} - sasToken {{{sre.storage.persistentdata.ingressSasToken}}} - authType SAS - containerName ingress - - - path: "/opt/mounts/ingress-mount.sh" - permissions: "0500" - content: | - if [ ! "$(df -h | grep $1)" ]; then - BLOBFUSE_CACHE_DIR="/tmp/blobfuse-cache-ingress" - rm -rf $BLOBFUSE_CACHE_DIR - mkdir -p $BLOBFUSE_CACHE_DIR - CACHE_SPACE_MB=$(echo "$(df -BM | grep /mnt | awk '{print $2}' | sed 's/M//') / 2" | bc) # set the cache size to half the size of /mnt which scales with VM size - /usr/bin/blobfuse $1 -o ro --tmp-path=$BLOBFUSE_CACHE_DIR --cache-size-mb=$CACHE_SPACE_MB --no-symlinks=true --config-file=/opt/mounts/ingress-credentials.secret --log-level=LOG_DEBUG -o attr_timeout=240 -o entry_timeout=240 -o negative_timeout=120 -o allow_other - fi - -# Set locale and timezone -locale: en_GB.UTF-8 -timezone: {{sre.time.timezone.linux}} - -# Set the NTP server -# By default we use Google's NTP servers which are incompatible with other servers due to leap-second smearing -ntp: - enabled: true - pools: - {{#shm.time.ntp.serverAddresses}} - - {{.}} - {{/shm.time.ntp.serverAddresses}} - -# Configure apt repositories -apt: - preserve_sources_list: true - sources: - microsoft-general.list: - source: "deb [arch=amd64] https://packages.microsoft.com/ubuntu/20.04/prod focal main" - keyid: BC528686B50D79E339D3721CEB3E94ADBE1229CF # Microsoft (Release signing) - -# Install necessary apt packages -packages: - - auditd - - blobfuse - - clamav - - clamav-base - - clamav-daemon - - clamav-freshclam - - clamav-unofficial-sigs - - docker.io - - docker-compose -package_update: true -package_upgrade: true - -# We know that exactly one data disk will be attached to this VM and it will be attached as lun1 -disk_setup: - /dev/disk/azure/scsi1/lun1: - table_type: gpt - layout: true - overwrite: true -fs_setup: - - device: /dev/disk/azure/scsi1/lun1 - partition: 1 - filesystem: ext4 -mounts: - - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"] - -# Set hostname -fqdn: {{sre.webapps.cocalc.fqdn}} -hostname: {{sre.webapps.cocalc.fqdn}} - -# Add the SRE admin (default) and cocalcdaemon users -users: - - default - - name: cocalcdaemon - lock_passwd: true # Lock the password to disable password login - sudo: false # This user will not have sudo privileges - -# Run other commands -runcmd: - # Suppress apt prompts and warning messages - - DEBIAN_FRONTEND=noninteractive - - export DEBIAN_FRONTEND - - # Clean up installation - - echo ">=== Cleaning up apt-get packages... ===<" - - apt update - - apt-get -y autoremove - - apt-get clean - - apt --fix-broken install - - # Ensure that auditd is running and enabled at startup - - echo ">=== Enabling auditd services... ===<" - - systemctl start auditd - - systemctl enable auditd - - sleep 20 - - systemctl status auditd - - # Configure ClamAV - - echo ">=== Configure ClamAV... ===<" - # Allow unlimited recursion when scanning - - sed -i 's/^MaxDirectoryRecursion .*/MaxDirectoryRecursion 0/' /etc/clamav/clamd.conf - # Enable ClamAV daemon - - systemctl enable clamav-daemon - # Enable ClamAV scan on access - - systemctl enable clamav-clamonacc - # Enable ClamAV daily scan - - systemctl enable clamav-clamdscan.timer - # Disable ClamAV database update on boot - - systemctl stop clamav-freshclam - - systemctl disable clamav-freshclam - - # Check server settings - - echo ">=== DNS ===<" - - /opt/configuration/set_dns.sh - - echo ">=== Hostname ===<" - - hostnamectl - - echo ">=== Date/time ===<" - - timedatectl - - # Configuring attached disks - - echo ">=== Configuring attached disks... ===<" - - mkdir -p /data/cocalc - - ls -alh /data/ - - # Ensure that Docker is running and enabled at startup - - echo ">=== Configuring Docker... ===<" - - systemctl enable docker - - systemctl start docker - - sleep 1m - - systemctl status docker - - docker --version - - docker-compose --version - - # Set up the cocalcdaemon user - - echo ">=== Configuring cocalcdaemon user... ===<" - - groupadd docker 2> /dev/null - - usermod -aG docker cocalcdaemon - - newgrp docker - - chown -R cocalcdaemon:cocalcdaemon /opt/cocalc - - ls -alh /opt/cocalc - - # Schedule mounting of data volume, allowing non-root users to specify 'allow_other' - - echo ">=== Configure ingress mount... ===<" - - grep -v "user_allow_other" /etc/fuse.conf > /etc/fuse.conf.tmp - - echo "user_allow_other" >> /etc/fuse.conf.tmp - - mv /etc/fuse.conf.tmp /etc/fuse.conf - - systemctl enable ingress.mount - - systemctl enable ingress.automount - - systemctl start ingress.mount - - sleep 20 - - systemctl status ingress.mount - - ls -alh /ingress - - # Deploy CoCalc using Docker - - echo ">=== Deploying CoCalc with Docker... ===<" - - su cocalcdaemon -c "docker-compose -f /opt/cocalc/docker-compose.yaml up -d" - # Wait for deployment to finish - - | - while true; do - if (curl --silent --insecure https://localhost | grep -q "Open CoCalc.*"); then - break - fi - sleep 5 - done - - docker-compose -f /opt/cocalc/docker-compose.yaml logs - # Print a final message - - echo "Deploying CoCalc with Docker is complete:" - - docker-compose -f /opt/cocalc/docker-compose.yaml ps - - -# Shutdown so that we can tell when the job has finished by polling the VM state -power_state: - mode: poweroff - message: "Shutting down as a signal that setup is finished" - timeout: 30 - condition: true diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml index f2afe406c1..890c39d5e3 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-codimd.mustache.yaml @@ -132,7 +132,7 @@ disk_setup: overwrite: true fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"] diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml index 8eefcfcbf6..b887420e2b 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-gitlab.mustache.yaml @@ -122,7 +122,7 @@ disk_setup: overwrite: true fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: - [/dev/disk/azure/scsi1/lun1-part1, /data, ext4, "defaults,nofail"] diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml index 16bcb94fa9..2a6c8f707b 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-postgres.mustache.yaml @@ -25,7 +25,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 - partition: 1 + partition: auto filesystem: ext4 mounts: diff --git a/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml b/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml index 12065238e6..45fa5f7ec9 100644 --- a/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml +++ b/deployment/secure_research_environment/cloud_init/cloud-init-srd.mustache.yaml @@ -12,7 +12,7 @@ disk_setup: fs_setup: - device: /dev/disk/azure/scsi1/lun1 filesystem: ext4 - partition: 1 + partition: auto # Note that we do not include the blobfuse mounts here as these are controlled by systemd mounts: @@ -155,17 +155,6 @@ write_files: options(repos = r) }) - - path: "/etc/skel/Desktop/CoCalc.desktop" - permissions: "0755" - content: | - [Desktop Entry] - Version=1.0 - Type=Link - Name=CoCalc - Comment= - Icon=/opt/icons/cocalc.png - URL=https://{{sre.webapps.cocalc.fqdn}} - - path: "/etc/skel/Desktop/CodiMD.desktop" permissions: "0755" content: | diff --git a/deployment/secure_research_environment/cloud_init/resources/cocalc_docker_compose.mustache.yaml b/deployment/secure_research_environment/cloud_init/resources/cocalc_docker_compose.mustache.yaml deleted file mode 100644 index 4f888d0d8d..0000000000 --- a/deployment/secure_research_environment/cloud_init/resources/cocalc_docker_compose.mustache.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: '3.7' -networks: - network_default: -services: - cocalc: - container_name: cocalc_cocalc_compose - image: sagemathinc/cocalc:{{sre.webapps.cocalc.dockerVersion}} - networks: - - network_default - ports: - - 443:443 - volumes: - - /data/cocalc:/projects - - /etc/localtime:/etc/localtime:ro - - /etc/timezone:/etc/timezone:ro - - /etc/pip.conf:/etc/pip.conf:ro - - /etc/R/Rprofile.site:/etc/R/Rprofile.site:ro - - /ingress:/data:ro - restart: always diff --git a/deployment/secure_research_environment/network_rules/sre-nsg-rules-webapps.json b/deployment/secure_research_environment/network_rules/sre-nsg-rules-webapps.json index d714002aec..b79fb2eab6 100644 --- a/deployment/secure_research_environment/network_rules/sre-nsg-rules-webapps.json +++ b/deployment/secure_research_environment/network_rules/sre-nsg-rules-webapps.json @@ -59,18 +59,6 @@ "sourceAddressPrefix": "*", "sourcePortRange": "*" }, - { - "name": "AllowPrivateDataEndpointsOutbound", - "access": "Allow", - "description": "Allow outbound connections to private endpoints in the VNet", - "destinationAddressPrefix": "{{sre.network.vnet.subnets.data.cidr}}", - "destinationPortRange": "*", - "direction": "Outbound", - "priority": 400, - "protocol": "*", - "sourceAddressPrefix": "{{sre.webapps.cocalc.ip}}", - "sourcePortRange": "*" - }, { "name": "AllowDNSOutbound", "access": "Allow", @@ -95,20 +83,6 @@ "sourceAddressPrefix": "{{sre.network.vnet.subnets.webapps.cidr}}", "sourcePortRange": "*" }, - {{#sre.repositories.network.cidr}} - { - "name": "AllowPackageRepositoriesOutbound", - "access": "Allow", - "description": "Allow package requests over http/https", - "destinationAddressPrefix": "{{sre.repositories.network.cidr}}", - "destinationPortRange": ["80", "443", "3128"], - "direction": "Outbound", - "priority": 1400, - "protocol": "*", - "sourceAddressPrefix": "{{sre.webapps.cocalc.ip}}", - "sourcePortRange": "*" - }, - {{/sre.repositories.network.cidr}} { "name": "AllowExternalNTPOutbound", "access": "Allow", diff --git a/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 b/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 index a47ddba25b..101c9f2f7b 100644 --- a/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 +++ b/deployment/secure_research_environment/remote/create_databases/scripts/Lockdown_Sql_Server.ps1 @@ -107,7 +107,7 @@ if ($operationFailed -Or (-Not $loginExists)) { # Create a DB user for each login group Write-Output "Ensuring that an SQL user exists for '$domainGroup' on: '$serverName'..." $sqlCommand = "IF NOT EXISTS(SELECT * FROM sys.database_principals WHERE name = '$domainGroup') CREATE USER [$domainGroup] FOR LOGIN [$domainGroup];" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query "$sqlCommand" -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Ensured that '$domainGroup' user exists on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -124,7 +124,7 @@ if ($operationFailed -Or (-Not $loginExists)) { foreach ($groupSchemaTuple in @(($DataAdminGroup, "data"), ($ResearchUsersGroup, "dbopublic"))) { $domainGroup, $schemaName = $groupSchemaTuple $sqlCommand = "IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = N'$schemaName') EXEC('CREATE SCHEMA $schemaName AUTHORIZATION [$domainGroup]');" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully ensured that '$schemaName' schema exists on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -154,7 +154,7 @@ if ($operationFailed -Or (-Not $loginExists)) { Write-Output " [x] Role $role not recognised!" continue } - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully gave '$domainGroup' $role permissions on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -171,7 +171,7 @@ if ($operationFailed -Or (-Not $loginExists)) { # ------------------------------------ Write-Output "Running T-SQL lockdown script on: '$serverName'..." $sqlCommand = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($ServerLockdownCommandB64)) - Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully ran T-SQL lockdown script on: '$serverName'" } else { @@ -187,7 +187,7 @@ if ($operationFailed -Or (-Not $loginExists)) { $windowsAdmin = "${serverName}\${VmAdminUsername}" Write-Output "Removing database access from $windowsAdmin on: '$serverName'..." $sqlCommand = "DROP USER IF EXISTS [$windowsAdmin]; IF EXISTS(SELECT * FROM master.dbo.syslogins WHERE loginname = '$windowsAdmin') DROP LOGIN [$windowsAdmin]" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully removed database access for $windowsAdmin on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete @@ -203,7 +203,7 @@ if ($operationFailed -Or (-Not $loginExists)) { # --------------------------------------------------------------------------------- Write-Output "Revoking sysadmin role from $DbAdminUsername on: '$serverName'..." $sqlCommand = "ALTER SERVER ROLE sysadmin DROP MEMBER $DbAdminUsername;" - Invoke-SqlCmd -ServerInstance $serverInstance -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true + Invoke-SqlCmd -ServerInstance $serverName -Credential $sqlAdminCredentials -QueryTimeout $connectionTimeoutInSeconds -Query $sqlCommand -TrustServerCertificate -ErrorAction SilentlyContinue -ErrorVariable sqlErrorMessage -OutputSqlErrors $true if ($? -And -Not $sqlErrorMessage) { Write-Output " [o] Successfully revoked sysadmin role on: '$serverName'" Start-Sleep -s 10 # allow time for the database action to complete diff --git a/deployment/secure_research_environment/setup/Apply_SRE_Network_Configuration.ps1 b/deployment/secure_research_environment/setup/Apply_SRE_Network_Configuration.ps1 index 401f21f0f0..1ad4e2f8fd 100644 --- a/deployment/secure_research_environment/setup/Apply_SRE_Network_Configuration.ps1 +++ b/deployment/secure_research_environment/setup/Apply_SRE_Network_Configuration.ps1 @@ -130,7 +130,7 @@ if (-not $config.sre.repositories.network.name) { # Set PyPI and CRAN locations on the SRD $null = Set-AzContext -SubscriptionId $config.sre.subscriptionName -ErrorAction Stop $scriptPath = Join-Path $PSScriptRoot ".." "remote" "network_configuration" "scripts" "update_mirror_settings.sh" -$repositoryFacingVms = Get-AzVM | Where-Object { ($_.ResourceGroupName -eq $config.sre.srd.rg) -or (($_.ResourceGroupName -eq $config.sre.webapps.rg) -and ($_.Name -eq $config.sre.webapps.cocalc.vmName)) } +$repositoryFacingVms = Get-AzVM | Where-Object { ($_.ResourceGroupName -eq $config.sre.srd.rg) foreach ($VM in $repositoryFacingVms) { Add-LogMessage -Level Info "Ensuring that PyPI and CRAN locations are set correctly on $($VM.Name)" $params = @{ diff --git a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 index a86077d1d1..51558b00b5 100644 --- a/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 +++ b/deployment/secure_research_environment/setup/Configure_External_DNS_Queries.ps1 @@ -26,7 +26,8 @@ $null = Set-AzContext -SubscriptionId $config.sre.subscriptionName -ErrorAction # -------------------------------------- $firewallRules = Get-JsonFromMustacheTemplate -TemplatePath (Join-Path $PSScriptRoot ".." ".." "safe_haven_management_environment" "network_rules" "shm-firewall-rules.json") -Parameters $config.shm -AsHashtable $allowedFqdns = @($firewallRules.applicationRuleCollections | ForEach-Object { $_.properties.rules.targetFqdns }) + - @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name }) + @(Get-PrivateDnsZones -ResourceGroupName $config.shm.network.vnet.rg -SubscriptionName $config.shm.subscriptionName | ForEach-Object { $_.Name }) + + @("docker.io") # List all unique FQDNs $allowedFqdns = $allowedFqdns | Where-Object { $_ -notlike "*-sb.servicebus.windows.net" } | # Remove AzureADConnect password reset endpoints diff --git a/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 b/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 index 0b7fbb2361..c7e8586193 100644 --- a/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 +++ b/deployment/secure_research_environment/setup/Setup_SRE_Databases.ps1 @@ -58,7 +58,6 @@ foreach ($databaseCfg in $config.sre.databases.instances) { } } else { Add-LogMessage -Level Warning "Database VM '$($databaseCfg.vmName)' already exists. Use the '-Redeploy' option if you want to remove the existing database and its data and deploy a new one." - continue } } diff --git a/deployment/secure_research_environment/setup/Setup_SRE_Key_Vault_And_Users.ps1 b/deployment/secure_research_environment/setup/Setup_SRE_Key_Vault_And_Users.ps1 index 7b403758a7..c025a22446 100644 --- a/deployment/secure_research_environment/setup/Setup_SRE_Key_Vault_And_Users.ps1 +++ b/deployment/secure_research_environment/setup/Setup_SRE_Key_Vault_And_Users.ps1 @@ -55,7 +55,6 @@ try { } # Other VMs $null = Resolve-KeyVaultSecret -VaultName $config.sre.keyVault.name -SecretName $config.sre.srd.adminPasswordSecretName -DefaultLength 20 -AsPlaintext - $null = Resolve-KeyVaultSecret -VaultName $config.sre.keyVault.name -SecretName $config.sre.webapps.cocalc.adminPasswordSecretName -DefaultLength 20 -AsPlaintext $null = Resolve-KeyVaultSecret -VaultName $config.sre.keyVault.name -SecretName $config.sre.webapps.codimd.adminPasswordSecretName -DefaultLength 20 -AsPlaintext $null = Resolve-KeyVaultSecret -VaultName $config.sre.keyVault.name -SecretName $config.sre.webapps.gitlab.adminPasswordSecretName -DefaultLength 20 -AsPlaintext Add-LogMessage -Level Success "Ensured that SRE VM admin passwords exist" diff --git a/deployment/secure_research_environment/setup/Setup_SRE_WebApp_Servers.ps1 b/deployment/secure_research_environment/setup/Setup_SRE_WebApp_Servers.ps1 index 08709e3fdc..fb220ff436 100644 --- a/deployment/secure_research_environment/setup/Setup_SRE_WebApp_Servers.ps1 +++ b/deployment/secure_research_environment/setup/Setup_SRE_WebApp_Servers.ps1 @@ -56,39 +56,6 @@ $cloudInitBasePath = Join-Path $PSScriptRoot ".." "cloud_init" -Resolve $ldapSearchUserDn = "CN=$($config.sre.users.serviceAccounts.ldapSearch.name),$($config.shm.domain.ous.serviceAccounts.path)" -# Deploy and configure CoCalc VM -# ------------------------------- -Add-LogMessage -Level Info "Constructing CoCalc cloud-init from template..." -# Load the cloud-init template then add resources and expand mustache placeholders -$cocalcCloudInitTemplate = Join-Path $cloudInitBasePath "cloud-init-cocalc.mustache.yaml" | Get-Item | Get-Content -Raw -$cocalcCloudInitTemplate = Expand-CloudInitResources -Template $cocalcCloudInitTemplate -ResourcePath (Join-Path $cloudInitBasePath "resources") -$cocalcCloudInitTemplate = Expand-CloudInitResources -Template $cocalcCloudInitTemplate -ResourcePath (Join-Path ".." ".." "common" "resources") -$cocalcCloudInitTemplate = Expand-MustacheTemplate -Template $cocalcCloudInitTemplate -Parameters $config -# Deploy CoCalc VM -$cocalcDataDisk = Deploy-ManagedDisk -Name "$($config.sre.webapps.cocalc.vmName)-DATA-DISK" -SizeGB $config.sre.webapps.cocalc.disks.data.sizeGb -Type $config.sre.webapps.cocalc.disks.data.type -ResourceGroupName $config.sre.webapps.rg -Location $config.sre.location -$params = @{ - AdminPassword = (Resolve-KeyVaultSecret -VaultName $config.sre.keyVault.name -SecretName $config.sre.webapps.cocalc.adminPasswordSecretName -DefaultLength 20) - AdminUsername = $vmAdminUsername - BootDiagnosticsAccount = $bootDiagnosticsAccount - CloudInitYaml = $cocalcCloudInitTemplate - DataDiskIds = @($cocalcDataDisk.Id) - ImageSku = $config.sre.webapps.cocalc.osVersion - Location = $config.sre.location - Name = $config.sre.webapps.cocalc.vmName - OsDiskSizeGb = $config.sre.webapps.cocalc.disks.os.sizeGb - OsDiskType = $config.sre.webapps.cocalc.disks.os.type - PrivateIpAddress = (Get-NextAvailableIpInRange -IpRangeCidr $config.sre.network.vnet.subnets.deployment.cidr -VirtualNetwork $vnet) - ResourceGroupName = $config.sre.webapps.rg - Size = $config.sre.webapps.cocalc.vmSize - Subnet = $deploymentSubnet -} -$cocalcVm = Deploy-LinuxVirtualMachine @params -# Change subnets and IP address while CoCalc VM is off then restart -Update-VMIpAddress -Name $cocalcVm.Name -ResourceGroupName $cocalcVm.ResourceGroupName -Subnet $webappsSubnet -IpAddress $config.sre.webapps.cocalc.ip -# Update DNS records for this VM -Update-VMDnsRecords -DcName $config.shm.dc.vmName -DcResourceGroupName $config.shm.dc.rg -BaseFqdn $config.sre.domain.fqdn -ShmSubscriptionName $config.shm.subscriptionName -VmHostname $config.sre.webapps.cocalc.hostname -VmIpAddress $config.sre.webapps.cocalc.ip - - # Deploy and configure CodiMD VM # ------------------------------ Add-LogMessage -Level Info "Constructing CodiMD cloud-init from template..." diff --git a/docs/source/deployment/build_srd_image.md b/docs/source/deployment/build_srd_image.md index 55e4e769e5..9482988160 100644 --- a/docs/source/deployment/build_srd_image.md +++ b/docs/source/deployment/build_srd_image.md @@ -108,7 +108,7 @@ PS> ./Provision_Compute_VM.ps1 -shmId ```{note} - Although the `./Provision_Compute_VM.ps1` script will finish running in a few minutes, the build itself will take several hours. -- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/verification/analyse_build.py`. +- We recommend **monitoring** the build by accessing the machine using `ssh` (the ssh info should be printed at the end of the Provision_Compute_VM.ps1 script) and either reading through the full build log at `/var/log/cloud-init-output.log` or running the summary script using `/opt/monitoring/analyse_build.py`. - **NB.** You will need to connect from an approved administrator IP address - **NB.** the VM will automatically shutdown at the end of the cloud-init process - if you want to analyse the build after this point, you will need to turn it back on in the `Azure` portal. ``` diff --git a/docs/source/deployment/deploy_sre.md b/docs/source/deployment/deploy_sre.md index e1dfbef923..74f909bb48 100644 --- a/docs/source/deployment/deploy_sre.md +++ b/docs/source/deployment/deploy_sre.md @@ -106,7 +106,43 @@ PS> ./Setup_SRE_Guacamole_Servers.ps1 -shmId -sreId
-Deploy web applications (CoCalc, CodiMD and GitLab) +Update SSL certificate + +![Powershell: five minutes](https://img.shields.io/static/v1?style=for-the-badge&logo=powershell&label=local&color=blue&message=five%20minutes) at {{file_folder}} `./deployment/secure_research_environment/setup` + +```powershell +PS> ./Update_SRE_SSL_Certificate.ps1 -shmId -sreId +``` + +- where `` is the {ref}`management environment ID ` for this SHM +- where `` is the {ref}`secure research environment ID ` for this SRE +- where `` is an email address that you want to be notified when certificates are close to expiry + +```{tip} +`./Update_SRE_RDS_SSL_Certificate.ps1` should be run again whenever you want to update the certificate for this SRE. +``` + +```{caution} +`Let's Encrypt` will only issue **5 certificates per week** for a particular host (e.g. `rdg-sre-sandbox.project.turingsafehaven.ac.uk`). +To reduce the number of calls to `Let's Encrypt`, the signed certificates are stored in the Key Vault for easy redeployment. +For production environments this should usually not be an issue. +``` + +````{important} +If you find yourself frequently redeploying a test environment and hit the `Let's Encrypt` certificate limit, you can can use: + +```powershell +> ./Update_SRE_RDS_SSL_Certificate.ps1 -dryRun $true +``` + +to use the `Let's Encrypt` staging server, which will issue certificates more frequently. +These certificates will **not** be trusted by your browser, and so should not be used in production. +```` + +
+ +
+Deploy web applications (CodiMD and GitLab) ```{include} snippets/07_deploy_webapps.partial.md :relative-images: @@ -237,13 +273,9 @@ If you see an error like the following when attempting to log in, it is likely t
```` -### {{snowflake}} Test CoCalc, CodiMD and GitLab servers +### {{snowflake}} Test CodiMD and GitLab servers - Connect to the remote desktop {ref}`using the instructions above ` -- Test `CoCalc` by clicking on the `CoCalc` desktop icon. - - This should open a web browser inside the remote desktop - - You will get a warning about a `Potential Security Risk` related to a self-signed certificate. It is safe to trust this by selecting `Advanced > Accept the risk and continue`. - - Create a new username and password and use this to log in. - Test `CodiMD` by clicking on the `CodiMD` desktop icon. - This should open a web browser inside the remote desktop - Log in with the short-form `username` of a user in the `SG Research Users` security group. diff --git a/docs/source/deployment/security_checklist.md b/docs/source/deployment/security_checklist.md index 3895be133a..d9de7e9ba4 100644 --- a/docs/source/deployment/security_checklist.md +++ b/docs/source/deployment/security_checklist.md @@ -640,7 +640,7 @@ To test all the above, you will need to act both as the {ref}`role_system_manage ``` ```{attention} -{{white_check_mark}} **Verify that:** software uploaded to the by a non-admin can be read by administrators +{{white_check_mark}} **Verify that:** software uploaded by a non-admin can be read by administrators ``` ```{attention} diff --git a/docs/source/design/architecture/sre_architecture.png b/docs/source/design/architecture/sre_architecture.png index e9a44d2ff7..4337f84318 100644 Binary files a/docs/source/design/architecture/sre_architecture.png and b/docs/source/design/architecture/sre_architecture.png differ diff --git a/docs/source/design/architecture/sre_details.md b/docs/source/design/architecture/sre_details.md index f6b05929fc..03adf9b945 100644 --- a/docs/source/design/architecture/sre_details.md +++ b/docs/source/design/architecture/sre_details.md @@ -14,7 +14,6 @@ This infrastructure comprises: - A file server to host the project data - A `Gitlab` server to provide source code management and version control - A `CodiMD` server for collaborative writing -- A `CoCalc` server for collaborative editing of computational notebook - `Apache Guacamole` provides a clientless remote desktop gateway to provide secure remote desktop access to the SRE resources. Hosting each secure project environment in its own resource group supports a clean lifecycle management process, making it easy to verifiably delete all project data and resources at the end of a project. diff --git a/docs/source/roles/researcher/user_guide.md b/docs/source/roles/researcher/user_guide.md index 402e908625..0daa143896 100644 --- a/docs/source/roles/researcher/user_guide.md +++ b/docs/source/roles/researcher/user_guide.md @@ -757,47 +757,6 @@ They will have to discuss whether this is an acceptable risk to the data securit You can make the process as easy as possible by providing as much information as possible about the code or data you'd like to bring into the environment and about how it is to be used. ``` -## {{couple}} Collaborate on code using CoCalc - -`CoCalc` is a collaborative calculation and data science environment. -It lets you work with others on projects, using `Jupyter`, `LaTeX`, `Octave`, `Python` or `R` in collaborative notebooks. - -The `CoCalc` instance within the SRE is the easiest way to work directly with others in your team (for example pair-programming) who might not be physically near you. -You do not need to worry about the security of the information you upload there as it is fully contained within the SRE and there is no access to the internet and / or external servers. - -```{important} -The `CoCalc` instance within the SRE is entirely separate from the https://cocalc.com service. -``` - -### {{unlock}} Access CoCalc - -You can access `CoCalc` from an internet browser in the SRD using the desktop shortcut. -The first time that you login, you will see a security warning. -This is expected, please click on `Advanced` and then `Accept the Risk and Continue`. - -```{image} user_guide/cocalc_security_warning.png -:alt: CoCalc security warning -:align: center -``` - -You will then get to the CoCalc homepage where you should click on `Sign In` - -```{image} user_guide/cocalc_homepage.png -:alt: CoCalc homepage -:align: center -``` - -You will need to create a new account. -You can use any username/password here - it is not connected to your main Safe Haven account. - -````{note} -Our example user, Ada Lovelace has used `ada.lovelace@projects.turingsafehaven.ac.uk` as her username and set her own password -```{image} user_guide/cocalc_account_creation.png -:alt: CoCalc account creation -:align: center -``` -```` - ## {{pill}} Versioning code using GitLab `GitLab` is a code hosting platform for version control and collaboration - similar to `GitHub`. diff --git a/docs/source/roles/system_manager/manage_data.md b/docs/source/roles/system_manager/manage_data.md index e517821476..2e2f9138e6 100644 --- a/docs/source/roles/system_manager/manage_data.md +++ b/docs/source/roles/system_manager/manage_data.md @@ -23,7 +23,7 @@ The following steps show how to generate a temporary write-only upload token tha - Click `Networking` under `Settings` and paste the data provider's IP address as one of those allowed under the `Firewall` header, then hit the save icon in the top left - From the `Overview` tab, click the link to `Containers` (in the middle of the page) - Click `ingress` -- Click `Shared access signature` under `Settings` and do the following: +- Click `Shared access tokens` under `Settings` and do the following: - Under `Permissions`, check these boxes: - `Write` - `List` @@ -70,7 +70,7 @@ The {ref}`role_system_manager` creates a time-limited and IP restricted link to - Ensure that the IP address of the person to receive the outputs is listed and enter it if not - Click `Containers` under `Data storage` - Click `egress` -- Click `Shared access signature` under `Settings` and do the following: +- Click `Shared access tokens` under `Settings` and do the following: - Under `Permissions`, check these boxes: - `Read` - `List` @@ -150,7 +150,6 @@ The disks covered by the protection for each SRE are the - GitLab data disk - CodiMD data disk -- CoCalc data disk - PostgreSQL data disk - MSSQL data disk diff --git a/docs/source/roles/system_manager/manage_users.md b/docs/source/roles/system_manager/manage_users.md index 7c832132ff..c83a2c80f3 100644 --- a/docs/source/roles/system_manager/manage_users.md +++ b/docs/source/roles/system_manager/manage_users.md @@ -137,6 +137,14 @@ The `DC1` is the source of truth for user details. If these details need to be c - Click on `Users` under `Manage` and search for the user - Confirm the user is no longer present +### {{x}} Automatically deleting all unassigned users + +In some situations, such as at the end of a project after an SRE has been torn down, you may want to remove all users from the SHM who are not assigned to the security group of any remaining attached SREs. + +- Ensure you have the same version of the Data Safe Haven repository as was used by your deployment team +- Open a `Powershell` terminal and navigate to the `deployment/administration` directory within the Data Safe Haven repository +- Run `./SHM_Delete_Unassigned_Users.ps1 -shmId ` (use the `-dryRun` flag to see who would get deleted with out performing the deletion) + ## {{calling}} Assign MFA licences ### {{hand}} Manually add licence to each user diff --git a/docs/source/roles/system_manager/manage_webapps.md b/docs/source/roles/system_manager/manage_webapps.md index 1869a82c24..9f739a7bb9 100644 --- a/docs/source/roles/system_manager/manage_webapps.md +++ b/docs/source/roles/system_manager/manage_webapps.md @@ -6,9 +6,9 @@ This document assumes that you already have access to a {ref}`Safe Haven Management (SHM) environment ` and one or more {ref}`Secure Research Environments (SREs) ` that are linked to it. ``` -During deployment of an SRE, distinct virtual machines are created to host each of the three standard web applications - `CoCalc`, `CodiMD`, and `Gitlab`. +During deployment of an SRE, distinct virtual machines are created to host each of the two standard web applications - `CodiMD` and `Gitlab`. -In principle, these should require no further direct interaction. Researchers using Secure Research Desktops will be able to interact with the servers through a web interface. `CoCalc` allows users to create their own user accounts, while `CodiMD` and `Gitlab` authenticate with the domain controller via LDAP. +In principle, these should require no further direct interaction. Researchers using Secure Research Desktops will be able to interact with the servers through a web interface. `CodiMD` and `Gitlab` authenticate with the domain controller via LDAP. However, it is possible for the virtual machine hosting the web app servers to successfully start without the web app servers themselves actually running. For example, Researchers using an `SRD` may find that the web apps are unavailable, or do not successfully authenticate log-in attempts. In such cases, command line access to the virtual machines hosting the web app servers may help to diagnose and resolve problems. @@ -19,7 +19,7 @@ In the rest of this document, `` is the {ref}`Secure Management Environm An initial step could be to check the build logs of the virtual machine to ascertain whether any clear errors occurred during the process (e.g. the installation of the server software may have failed). - From the `Azure` portal, navigate to the web app resource group `RG_SHM__SRE__WEBAPPS`. -- Click on the relevant VM (e.g. `COCALC-SRE-`) +- Click on the relevant VM (e.g. `CODIMD-SRE-`) - From the menu on the left, scroll down to the `Help` section and select `Boot diagnostics` - Click `Serial log` to access a full text log of the booting up of the VM. diff --git a/environment_configs/package_lists/allowlist-core-r-cran-tier3.list b/environment_configs/package_lists/allowlist-core-r-cran-tier3.list index 23977ee481..7fc527f500 100644 --- a/environment_configs/package_lists/allowlist-core-r-cran-tier3.list +++ b/environment_configs/package_lists/allowlist-core-r-cran-tier3.list @@ -1,3 +1,4 @@ +arrow BiocManager car caret diff --git a/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list b/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list index e2dc7471c2..e69de29bb2 100644 --- a/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list +++ b/environment_configs/package_lists/allowlist-extra-r-cran-tier3.list @@ -1 +0,0 @@ -arrow diff --git a/environment_configs/package_lists/allowlist-full-python-pypi-tier3.list b/environment_configs/package_lists/allowlist-full-python-pypi-tier3.list index 0bfe16e893..e160c08250 100644 --- a/environment_configs/package_lists/allowlist-full-python-pypi-tier3.list +++ b/environment_configs/package_lists/allowlist-full-python-pypi-tier3.list @@ -28,7 +28,6 @@ arviz asn1crypto astor astroid -astropy asttokens astunparse async_generator @@ -38,6 +37,7 @@ asyncio asynctest atomicwrites attrs +autocommand autograd autograd-gamma Automat @@ -75,6 +75,7 @@ Bottleneck bpemb branca Brotli +brotlicffi bson build bulwark @@ -105,7 +106,6 @@ comm commonmark confection configparser -confuse conllu cons constantly @@ -241,6 +241,7 @@ importlib importlib-metadata importlib-resources incremental +inflect iniconfig installer interface-meta @@ -248,11 +249,19 @@ ipaddress ipykernel ipython ipython_genutils -ipywidgets isort itsdangerous Janome +jaraco.apt jaraco.classes +jaraco.collections +jaraco.context +jaraco.functools +jaraco.structures +jaraco.text +jaraco.timing +jaraco.ui +jaraco.windows jarowinkler jax jedi @@ -280,7 +289,6 @@ jupyterlab jupyterlab-launcher jupyterlab-pygments jupyterlab-server -jupyterlab-widgets keras Keras-Applications keras-nightly @@ -458,7 +466,6 @@ pycurl pydantic pydantic-core pydot -pyerfa pyflakes pyFUME Pygments @@ -562,6 +569,7 @@ Send2Trash sentencepiece setuptools setuptools-git +setuptools-scm shapely shellingham simpful @@ -610,6 +618,7 @@ tabulate tangled-up-in-unicode tblib tdda +tempora tenacity tensorboard tensorboard-data-server @@ -684,7 +693,6 @@ webencodings websocket-client Werkzeug wheel -widgetsnbextension Wikipedia-API win_unicode_console win32-setctime @@ -704,7 +712,9 @@ y-py yarl yaspin ydata-profiling +yg.lockfile ypy-websocket +zc.lockfile zict zipfile36 zipp diff --git a/environment_configs/package_lists/allowlist-full-r-cran-tier3.list b/environment_configs/package_lists/allowlist-full-r-cran-tier3.list index 9e12103115..4b1441c1af 100644 --- a/environment_configs/package_lists/allowlist-full-r-cran-tier3.list +++ b/environment_configs/package_lists/allowlist-full-r-cran-tier3.list @@ -156,7 +156,6 @@ gdtools generics geojson geojsonio -geojsonlint geojsonsf geometries geometry @@ -235,7 +234,6 @@ jqr jquerylib jsonify jsonlite -jsonvalidate kdensity kernlab KernSmooth @@ -382,6 +380,7 @@ quadprog quanteda quantmod quantreg +QuickJSR qvcalc R.cache R.methodsS3 diff --git a/environment_configs/sre_bluet1guac_core_config.json b/environment_configs/sre_bluet1guac_core_config.json index 2454eb73e8..365ae14a8c 100644 --- a/environment_configs/sre_bluet1guac_core_config.json +++ b/environment_configs/sre_bluet1guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet2guac_core_config.json b/environment_configs/sre_bluet2guac_core_config.json index 2ada45f6ec..15d96d2cda 100644 --- a/environment_configs/sre_bluet2guac_core_config.json +++ b/environment_configs/sre_bluet2guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_bluet3guac_core_config.json b/environment_configs/sre_bluet3guac_core_config.json index c14e6af1f0..eff5d1e24c 100644 --- a/environment_configs/sre_bluet3guac_core_config.json +++ b/environment_configs/sre_bluet3guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.240"], diff --git a/environment_configs/sre_greent1guac_core_config.json b/environment_configs/sre_greent1guac_core_config.json index e78af97072..10cad425c3 100644 --- a/environment_configs/sre_greent1guac_core_config.json +++ b/environment_configs/sre_greent1guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent2guac_core_config.json b/environment_configs/sre_greent2guac_core_config.json index f0e726431c..091a74e605 100644 --- a/environment_configs/sre_greent2guac_core_config.json +++ b/environment_configs/sre_greent2guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.253"], diff --git a/environment_configs/sre_greent3guac_core_config.json b/environment_configs/sre_greent3guac_core_config.json index 95f59302f5..a3c0c4c57e 100644 --- a/environment_configs/sre_greent3guac_core_config.json +++ b/environment_configs/sre_greent3guac_core_config.json @@ -8,7 +8,7 @@ "outboundInternetAccess": "default", "computeVmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "remoteDesktopProvider": "ApacheGuacamole", "dataAdminIpAddresses": ["193.60.220.240"], diff --git a/pyproject.toml b/pyproject.toml index e033e8459b..45d64e4052 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,11 +152,15 @@ module = [ "cryptography.*", "dns.*", "msal.*", + "numpy.*", + "pandas.*", "psycopg.*", "pulumi.*", "pulumi_azure_native.*", + "pymssql.*", "rich.*", "simple_acme_dns.*", + "sklearn.*", "typer.*", "websocket.*", ] diff --git a/tests/resources/shm_blue_full_config.json b/tests/resources/shm_blue_full_config.json index 14d42975fb..7e521011d1 100644 --- a/tests/resources/shm_blue_full_config.json +++ b/tests/resources/shm_blue_full_config.json @@ -142,6 +142,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", diff --git a/tests/resources/shm_green_full_config.json b/tests/resources/shm_green_full_config.json index 2d72b0a912..c5326b643a 100644 --- a/tests/resources/shm_green_full_config.json +++ b/tests/resources/shm_green_full_config.json @@ -142,6 +142,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", diff --git a/tests/resources/sre_bluet1guac_full_config.json b/tests/resources/sre_bluet1guac_full_config.json index 0aed7c6062..8ee042161e 100644 --- a/tests/resources/sre_bluet1guac_full_config.json +++ b/tests/resources/sre_bluet1guac_full_config.json @@ -143,6 +143,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", @@ -1093,15 +1100,8 @@ "instances": [ { "adminPasswordSecretName": "sre-t1guac-vm-admin-password-mssql", - "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql", "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-mssql", - "vmName": "MSSQL-T1GUAC", - "type": "MSSQL", - "ip": "10.151.3.4", - "port": "1433", - "sku": "sqldev-gen2", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-mssql", "disks": { "data": { "sizeGb": "1024", @@ -1112,19 +1112,19 @@ "type": "Standard_LRS" } }, - "enableSSIS": true + "enableSSIS": true, + "ip": "10.151.3.4", + "port": "1433", + "sku": "sqldev-gen2", + "subnet": "databases", + "type": "MSSQL", + "vmName": "MSSQL-T1GUAC", + "vmSize": "Standard_DS2_v2" }, { "adminPasswordSecretName": "sre-t1guac-vm-admin-password-postgresql", - "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql", "dbAdminPasswordSecretName": "sre-t1guac-db-admin-password-postgresql", - "vmName": "PSTGRS-T1GUAC", - "type": "PostgreSQL", - "ip": "10.151.3.5", - "port": "5432", - "sku": "Ubuntu-latest", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t1guac-db-admin-username-postgresql", "disks": { "data": { "sizeGb": "1024", @@ -1134,7 +1134,14 @@ "sizeGb": "128", "type": "Standard_LRS" } - } + }, + "ip": "10.151.3.5", + "port": "5432", + "sku": "Ubuntu-latest", + "subnet": "databases", + "type": "PostgreSQL", + "vmName": "PSTGRS-T1GUAC", + "vmSize": "Standard_DS2_v2" } ], "rg": "RG_SHM_BLUE_SRE_T1GUAC_DATABASES" @@ -1284,7 +1291,7 @@ "rg": "RG_SHM_BLUE_SRE_T1GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "vmSizeDefault": "Standard_D2s_v3" }, @@ -1387,26 +1394,6 @@ } }, "webapps": { - "cocalc": { - "adminPasswordSecretName": "sre-t1guac-vm-admin-password-cocalc", - "disks": { - "data": { - "sizeGb": "512", - "type": "Standard_LRS" - }, - "os": { - "sizeGb": "32", - "type": "Standard_LRS" - } - }, - "dockerVersion": "latest", - "fqdn": "COCALC.t1guac.blue.develop.turingsafehaven.ac.uk", - "hostname": "COCALC", - "ip": "10.151.5.7", - "osVersion": "Ubuntu-latest", - "vmName": "COCALC-SRE-T1GUAC", - "vmSize": "Standard_D2s_v3" - }, "codimd": { "adminPasswordSecretName": "sre-t1guac-vm-admin-password-codimd", "codimd": { diff --git a/tests/resources/sre_bluet3guac_full_config.json b/tests/resources/sre_bluet3guac_full_config.json index de66cdf330..a2f21eb68a 100644 --- a/tests/resources/sre_bluet3guac_full_config.json +++ b/tests/resources/sre_bluet3guac_full_config.json @@ -143,6 +143,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", @@ -1284,7 +1291,7 @@ "rg": "RG_SHM_BLUE_SRE_T3GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "vmSizeDefault": "Standard_D2s_v3" }, @@ -1387,26 +1394,6 @@ } }, "webapps": { - "cocalc": { - "adminPasswordSecretName": "sre-t3guac-vm-admin-password-cocalc", - "disks": { - "data": { - "sizeGb": "512", - "type": "Standard_LRS" - }, - "os": { - "sizeGb": "32", - "type": "Standard_LRS" - } - }, - "dockerVersion": "latest", - "fqdn": "COCALC.t3guac.blue.develop.turingsafehaven.ac.uk", - "hostname": "COCALC", - "ip": "10.153.5.7", - "osVersion": "Ubuntu-latest", - "vmName": "COCALC-SRE-T3GUAC", - "vmSize": "Standard_D2s_v3" - }, "codimd": { "adminPasswordSecretName": "sre-t3guac-vm-admin-password-codimd", "codimd": { diff --git a/tests/resources/sre_greent2guac_full_config.json b/tests/resources/sre_greent2guac_full_config.json index 33fa1bdc21..165e24acda 100644 --- a/tests/resources/sre_greent2guac_full_config.json +++ b/tests/resources/sre_greent2guac_full_config.json @@ -143,6 +143,13 @@ "217.196.149.55", "91.189.91.38", "91.189.91.39", + "91.189.91.48", + "91.189.91.49", + "91.189.91.81", + "91.189.91.82", + "91.189.91.83", + "185.125.190.17", + "185.125.190.18", "185.125.190.36", "185.125.190.39", "103.21.244.0/22", @@ -1146,15 +1153,8 @@ "instances": [ { "adminPasswordSecretName": "sre-t2guac-vm-admin-password-mssql", - "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-mssql", "dbAdminPasswordSecretName": "sre-t2guac-db-admin-password-mssql", - "vmName": "MSSQL-T2GUAC", - "type": "MSSQL", - "ip": "10.152.3.4", - "port": "1433", - "sku": "sqldev-gen2", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-mssql", "disks": { "data": { "sizeGb": "1024", @@ -1165,19 +1165,19 @@ "type": "Standard_LRS" } }, - "enableSSIS": true + "enableSSIS": true, + "ip": "10.152.3.4", + "port": "1433", + "sku": "sqldev-gen2", + "subnet": "databases", + "type": "MSSQL", + "vmName": "MSSQL-T2GUAC", + "vmSize": "Standard_DS2_v2" }, { "adminPasswordSecretName": "sre-t2guac-vm-admin-password-postgresql", - "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-postgresql", "dbAdminPasswordSecretName": "sre-t2guac-db-admin-password-postgresql", - "vmName": "PSTGRS-T2GUAC", - "type": "PostgreSQL", - "ip": "10.152.3.5", - "port": "5432", - "sku": "Ubuntu-latest", - "subnet": "databases", - "vmSize": "Standard_DS2_v2", + "dbAdminUsernameSecretName": "sre-t2guac-db-admin-username-postgresql", "disks": { "data": { "sizeGb": "1024", @@ -1187,7 +1187,14 @@ "sizeGb": "128", "type": "Standard_LRS" } - } + }, + "ip": "10.152.3.5", + "port": "5432", + "sku": "Ubuntu-latest", + "subnet": "databases", + "type": "PostgreSQL", + "vmName": "PSTGRS-T2GUAC", + "vmSize": "Standard_DS2_v2" } ], "rg": "RG_SHM_GREEN_SRE_T2GUAC_DATABASES" @@ -1337,7 +1344,7 @@ "rg": "RG_SHM_GREEN_SRE_T2GUAC_COMPUTE", "vmImage": { "type": "Ubuntu", - "version": "20.04.2023031401" + "version": "20.04.2023082900" }, "vmSizeDefault": "Standard_D2s_v3" }, @@ -1440,26 +1447,6 @@ } }, "webapps": { - "cocalc": { - "adminPasswordSecretName": "sre-t2guac-vm-admin-password-cocalc", - "disks": { - "data": { - "sizeGb": "512", - "type": "Standard_LRS" - }, - "os": { - "sizeGb": "32", - "type": "Standard_LRS" - } - }, - "dockerVersion": "latest", - "fqdn": "COCALC.t2guac.green.develop.turingsafehaven.ac.uk", - "hostname": "COCALC", - "ip": "10.152.5.7", - "osVersion": "Ubuntu-latest", - "vmName": "COCALC-SRE-T2GUAC", - "vmSize": "Standard_D2s_v3" - }, "codimd": { "adminPasswordSecretName": "sre-t2guac-vm-admin-password-codimd", "codimd": { diff --git a/tests/srd_smoke_tests/test_packages_installed_python.py b/tests/srd_smoke_tests/test_packages_installed_python.py index a01a2c44c1..d91d3238f7 100644 --- a/tests/srd_smoke_tests/test_packages_installed_python.py +++ b/tests/srd_smoke_tests/test_packages_installed_python.py @@ -4,6 +4,7 @@ import subprocess import sys import warnings + import pkg_resources versions = { @@ -22,7 +23,10 @@ ] # For these packages we check for an executable as they are not importable -NON_IMPORTABLE_PACKAGES = {"repro-catalogue": "catalogue"} +NON_IMPORTABLE_PACKAGES = { + "pip-tools": "pip-compile", + "repro-catalogue": "catalogue", +} # Some packages are imported using a different name than they `pip install` with IMPORTABLE_NAMES = {