Skip to content

Commit

Permalink
Merge pull request #5 from craddm/develop
Browse files Browse the repository at this point in the history
update fix-error-messages branch
  • Loading branch information
craddm authored Oct 2, 2023
2 parents 283f0e9 + cfc3dc2 commit e5f7642
Show file tree
Hide file tree
Showing 59 changed files with 266 additions and 640 deletions.
2 changes: 1 addition & 1 deletion .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \

# Set package versions
ARG AZURE_CLI_VERSION="2.42.0"
ARG PWSH_VERSION="7.3.2"
ARG PWSH_VERSION="7.3.6"

# Set up TARGETARCH variable to use to pull the right binaries for the current architecture.
ARG TARGETARCH
Expand Down
2 changes: 1 addition & 1 deletion CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,5 @@
# We only plan to require code owner review for
# main and other branches that may be deployed from.
# Note: /dir/ applies to directory and all subdirectories
/deployment/ @martintoreilly @jemrobinson @JimMadge
/deployment/ @martintoreilly @jemrobinson @JimMadge @craddm
/docs/ @martintoreilly @jemrobinson @JimMadge @craddm @edwardchalstrey1
1 change: 1 addition & 0 deletions VERSIONING.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ We usually deploy the latest available version of the Data Safe Haven for each o
| December 2021 | DSG 2021-12 | [v3.3.1](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v3.3.1) |
| December 2022 | DSG 2022-12 | [v4.0.2](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.2) |
| February 2023 | DSG 2023-02 | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3) |
| May 2023 | DSG 2023-05 | [v4.0.3](https://github.com/alan-turing-institute/data-safe-haven/releases/tag/v4.0.3) |

## Versions that have undergone formal security evaluation

Expand Down
2 changes: 1 addition & 1 deletion deployment/CheckRequirements.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ param (
Import-Module $PSScriptRoot/common/Logging -Force -ErrorAction Stop

# Requirements
$PowershellSupportedVersion = "7.3.2"
$PowershellSupportedVersion = "7.3.6"
$ModuleVersionRequired = @{
"Az.Accounts" = @("ge", "2.11.1")
"Az.Automation" = @("ge", "1.9.0")
Expand Down
33 changes: 33 additions & 0 deletions deployment/administration/SHM_Delete_Unassigned_Users.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
param(
[Parameter(Mandatory = $true, HelpMessage = "Enter SHM ID (e.g. use 'testa' for Turing Development Safe Haven A)")]
[string]$shmId,
[Parameter(Mandatory = $false, HelpMessage = "No-op mode which will not remove anything")]
[Switch]$dryRun
)

Import-Module Az.Accounts -ErrorAction Stop
Import-Module $PSScriptRoot/../common/AzureCompute -Force -ErrorAction Stop
Import-Module $PSScriptRoot/../common/Configuration -Force -ErrorAction Stop
Import-Module $PSScriptRoot/../common/Logging -Force -ErrorAction Stop

# Get config
# -------------------------------
$config = Get-ShmConfig -shmId $shmId
$originalContext = Get-AzContext

# Delete users not currently in a security group
# ----------------------------------------------
$null = Set-AzContext -SubscriptionId $config.subscriptionName -ErrorAction Stop
$script = "remote/Delete_Unassigned_Users.ps1"

# Passing a param to a remote script requires it to be a string
if ($dryRun.IsPresent) {
Add-LogMessage -Level Info "Listing users not assigned to any security group from $($config.dc.vmName)..."
$params = @{dryRun = "yes" }
} else {
Add-LogMessage -Level Info "Deleting users not assigned to any security group from $($config.dc.vmName)..."
$params = @{dryRun = "no" }
}
$result = Invoke-RemoteScript -Shell "PowerShell" -ScriptPath $script -VMName $config.dc.vmName -ResourceGroupName $config.dc.rg -Parameter $params

$null = Set-AzContext -Context $originalContext -ErrorAction Stop
33 changes: 33 additions & 0 deletions deployment/administration/remote/Delete_Unassigned_Users.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
param(
[Parameter(Mandatory = $true, HelpMessage = "yes/no determines whether users should actually be deleted")]
[string]$dryRun
)

# Extract list of users
$userOuPath = (Get-ADObject -Filter * | Where-Object { $_.Name -eq "Safe Haven Research Users" }).DistinguishedName
$users = Get-ADUser -Filter * -SearchBase "$userOuPath" -Properties *
foreach ($user in $users) {
$groupName = ($user | Select-Object -ExpandProperty MemberOf | ForEach-Object { (($_ -Split ",")[0] -Split "=")[1] }) -join "|"
if (!($groupName)) {
$name = $user.SamAccountName
if ($dryRun -eq "yes") {
Write-Output "User $name would be deleted by this action"
} else {
Write-Output "Deleting $name"
Remove-ADUser -Identity $name -Confirm:$false
}
}
}

# Force sync with AzureAD. It will still take around 5 minutes for changes to propagate
if ($dryRun -eq "no") {
Write-Output "Synchronising locally Active Directory with Azure"
try {
Import-Module -Name "C:\Program Files\Microsoft Azure AD Sync\Bin\ADSync" -ErrorAction Stop
Start-ADSyncSyncCycle -PolicyType Delta
} catch [System.IO.FileNotFoundException] {
Write-Output "Skipping as Azure AD Sync is not installed"
} catch {
Write-Output "Unable to run Azure Active Directory synchronisation!"
}
}
25 changes: 3 additions & 22 deletions deployment/common/Configuration.psm1
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ function Get-ShmConfig {
) # *-jobruntimedata-prod-su1.azure-automation.net
linux = (
@("72.32.157.246", "87.238.57.227", "147.75.85.69", "217.196.149.55") + # apt.postgresql.org
@("91.189.91.38", "91.189.91.39", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
@("91.189.91.38", "91.189.91.39", "91.189.91.48", "91.189.91.49", "91.189.91.81", "91.189.91.82", "91.189.91.83", "185.125.190.17", "185.125.190.18", "185.125.190.36", "185.125.190.39") + # archive.ubuntu.com, changelogs.ubuntu.com, security.ubuntu.com
$cloudFlareIpAddresses + # database.clamav.net, packages.gitlab.com and qgis.org use Cloudflare
$cloudFrontIpAddresses + # packages.gitlab.com uses Cloudfront to host its Release file
@("104.131.190.124") + # dbeaver.io
Expand Down Expand Up @@ -729,7 +729,6 @@ function Get-SreConfig {
}
}


# Firewall config
# ---------------
$config.sre.firewall = [ordered]@{
Expand Down Expand Up @@ -928,28 +927,10 @@ function Get-SreConfig {
$config.sre.remoteDesktop.networkRules.includeAzurePlatformDnsRule = ($config.sre.remoteDesktop.networkRules.outboundInternet -ne "Allow")


# CoCalc, CodiMD and Gitlab servers
# ---------------------------------
# CodiMD and Gitlab servers
# -------------------------
$config.sre.webapps = [ordered]@{
rg = "$($config.sre.rgPrefix)_WEBAPPS".ToUpper()
cocalc = [ordered]@{
adminPasswordSecretName = "$($config.sre.shortName)-vm-admin-password-cocalc"
dockerVersion = "latest"
hostname = "COCALC"
vmSize = "Standard_D2s_v3"
ip = Get-NextAvailableIpInRange -IpRangeCidr $config.sre.network.vnet.subnets.webapps.cidr -Offset 7
osVersion = "Ubuntu-latest"
disks = [ordered]@{
data = [ordered]@{
sizeGb = "512"
type = $config.sre.diskTypeDefault
}
os = [ordered]@{
sizeGb = "32"
type = $config.sre.diskTypeDefault
}
}
}
codimd = [ordered]@{
adminPasswordSecretName = "$($config.sre.shortName)-vm-admin-password-codimd"
hostname = "CODIMD"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ disk_setup:

fs_setup:
- device: /dev/disk/azure/scsi1/lun1
partition: 1
partition: auto
filesystem: ext4

mounts:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ disk_setup:

fs_setup:
- device: /dev/disk/azure/scsi1/lun1
partition: 1
partition: auto
filesystem: ext4

mounts:
Expand Down Expand Up @@ -122,7 +122,7 @@ write_files:
- path: "/etc/cron.d/pull-from-internet"
permissions: "0644"
content: |
# External update (rsync from CRAN) every 6 hours
# External update from PyPi every 6 hours
0 */6 * * * mirrordaemon ~mirrordaemon/pull_from_internet.sh
- path: "/etc/cron.d/pull-then-push"
Expand Down Expand Up @@ -313,7 +313,7 @@ runcmd:

# Install bandersnatch with pip
- echo ">=== Installing bandersnatch... ===<"
- pip3 install bandersnatch==4.2.0
- pip3 install bandersnatch==4.2.0 packaging==21.3
- echo "Using bandersnatch from '$(which bandersnatch)'"

# Initialise allowlist if appropriate
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ disk_setup:

fs_setup:
- device: /dev/disk/azure/scsi1/lun1
partition: 1
partition: auto
filesystem: ext4

mounts:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ disk_setup:

fs_setup:
- device: /dev/disk/azure/scsi1/lun1
partition: 1
partition: auto
filesystem: ext4

mounts:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,6 @@ write_files:
content: |
{{packages-python.yaml}}
- path: "/opt/build/pyenv/pyproject_template.toml"
permissions: "0400"
content: |
{{pyenv_pyproject_template.toml}}
- path: "/opt/build/rstudio.debinfo"
permissions: "0400"
content: |
Expand Down Expand Up @@ -257,8 +252,6 @@ runcmd:
# Get icons for webapps
- echo "Downloading icons..."
- mkdir -p /opt/icons
- wget https://raw.githubusercontent.com/sagemathinc/cocalc/master/src/packages/assets/cocalc-icon.svg -O /tmp/cocalc-icon.svg || die "Could not find CoCalc icon!"
- convert -density 983 -transparent white /tmp/cocalc-icon.svg /opt/icons/cocalc.png || die "Could not convert CoCalc icon!" # this should give a 1024x1024 png
- wget https://raw.githubusercontent.com/hackmdio/codimd/develop/public/favicon.png -O /opt/icons/codimd.png || die "Could not find CodiMD icon!"
- wget https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png -O /opt/icons/gitlab.png || die "Could not find GitLab icon!"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,11 +114,6 @@ write_files:
content: |
{{packages-python.yaml}}
- path: "/opt/build/pyenv/pyproject_template.toml"
permissions: "0400"
content: |
{{pyenv_pyproject_template.toml}}
- path: "/opt/build/rstudio.debinfo"
permissions: "0400"
content: |
Expand Down Expand Up @@ -260,8 +255,6 @@ runcmd:
# Get icons for webapps
- echo "Downloading icons..."
- mkdir -p /opt/icons
- wget https://raw.githubusercontent.com/sagemathinc/cocalc/master/src/packages/assets/cocalc-icon.svg -O /tmp/cocalc-icon.svg || die "Could not find CoCalc icon!"
- convert -density 983 -transparent white /tmp/cocalc-icon.svg /opt/icons/cocalc.png || die "Could not convert CoCalc icon!" # this should give a 1024x1024 png
- wget https://raw.githubusercontent.com/hackmdio/codimd/develop/public/favicon.png -O /opt/icons/codimd.png || die "Could not find CodiMD icon!"
- wget https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png -O /opt/icons/gitlab.png || die "Could not find GitLab icon!"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,6 @@ write_files:
content: |
{{packages-python.yaml}}
- path: "/opt/build/pyenv/pyproject_template.toml"
permissions: "0400"
content: |
{{pyenv_pyproject_template.toml}}
- path: "/opt/build/rbase.debinfo"
permissions: "0400"
content: |
Expand Down Expand Up @@ -266,8 +261,6 @@ runcmd:
# Get icons for webapps
- echo "Downloading icons..."
- mkdir -p /opt/icons
- wget https://raw.githubusercontent.com/sagemathinc/cocalc/master/src/packages/assets/cocalc-icon.svg -O /tmp/cocalc-icon.svg || die "Could not find CoCalc icon!"
- convert -density 983 -transparent white /tmp/cocalc-icon.svg /opt/icons/cocalc.png || die "Could not convert CoCalc icon!" # this should give a 1024x1024 png
- wget https://raw.githubusercontent.com/hackmdio/codimd/develop/public/favicon.png -O /opt/icons/codimd.png || die "Could not find CodiMD icon!"
- wget https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png -O /opt/icons/gitlab.png || die "Could not find GitLab icon!"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ if [ $# -ne 1 ]; then
fi
PYTHON_VERSION=$1
PYENV_ROOT="$(pyenv root)"
PYPROJECT_TOML="/opt/build/python-${PYTHON_VERSION}-pyproject.toml"
MONITORING_LOG="/opt/monitoring/python-${PYTHON_VERSION}-package-versions.log"
REQUIREMENTS_TXT="/opt/build/python-${PYTHON_VERSION}-requirements.txt"
REQUESTED_PACKAGE_LIST="/opt/build/packages/packages-python-${PYTHON_VERSION}.list"
Expand All @@ -27,24 +26,19 @@ echo "Installed $(${EXE_PATH}/python --version)"
# Install and upgrade installation prerequisites
# ----------------------------------------------
echo "Installing and upgrading installation prerequisites for Python ${PYTHON_VERSION}..."
${EXE_PATH}/pip install --upgrade pip poetry
${EXE_PATH}/pip install --upgrade pip pip-tools setuptools


# Solve dependencies and install using poetry
# -------------------------------------------
echo "Installing packages with poetry..."
${EXE_PATH}/poetry config virtualenvs.create false
${EXE_PATH}/poetry config virtualenvs.in-project true
rm poetry.lock pyproject.toml 2> /dev/null
sed -e "s/PYTHON_VERSION/$PYTHON_VERSION/" /opt/build/pyenv/pyproject_template.toml > $PYPROJECT_TOML
ln -s $PYPROJECT_TOML pyproject.toml
${EXE_PATH}/poetry add $(tr '\n' ' ' < $REQUIREMENTS_TXT) || exit 3
# Solve dependencies and write package versions to monitoring log
# ---------------------------------------------------------------
echo "Determining package versions with pip-compile..."
${EXE_PATH}/pip-compile -o "$MONITORING_LOG" "$REQUIREMENTS_TXT"


# Write package versions to monitoring log
# ----------------------------------------
${EXE_PATH}/poetry show > $MONITORING_LOG
${EXE_PATH}/poetry show --tree >> $MONITORING_LOG
# Install pinned packages using pip
# ---------------------------------
echo "Installing packages with pip..."
${EXE_PATH}/pip install -r "$MONITORING_LOG"


# Run any post-install commands
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
hash: c105a5286861ff1f312ab699d5810af7dc0e668917afb1f80c4eda3a58bb8d34
version: 1.45.1
hash: 6c75efb8596d25152bc44f6f59a4b96cd013219b0c87ae2365ca8f69fe29e206
version: 1.46.0
debfile: azuredatastudio-linux-|VERSION|.deb
remote: https://sqlopsbuilds.azureedge.net/stable/88c21b1725a3e79440027bdb7b5a55fb036be0e2/|DEBFILE|
remote: https://sqlopsbuilds.azureedge.net/stable/39449bbe88a0bc4092c9b205cad10d0a556beffd/|DEBFILE|
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
hash: 49e24a6956f9f12ffeded493f571cd39f3e6c89411fc60d3bb228661793320da
version: 2023.06.1-524
hash: 981fcbb30d5dea283188fcef1a9cdf177bb51b83884a63fd3c9c224d1486b01e
version: 2023.06.2-561
debfile: rstudio-|VERSION|-amd64.deb
remote: https://download1.rstudio.org/electron/focal/amd64/|DEBFILE|
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
hash: c030ec8338f1c76b3ae27997ec4411a0af43b2367dedb3d48e95c319b5425698
version: 2023.06.1-524
hash: bb6b3c21510abb18fd6e697567d7ff3d4135bf7980cf25536753e9ceac60c82c
version: 2023.06.2-561
debfile: rstudio-|VERSION|-amd64.deb
remote: https://download1.rstudio.org/electron/jammy/amd64/|DEBFILE|
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,8 @@ packages:
pathos:
pg8000:
Pillow:
pip-tools:
plotly:
poetry: # also used by installation scripts
"all": [">1.0.0"] # increase solver flexibility
prophet:
psycopg2:
pydot:
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
arrow
BiocManager
caret
csv
Expand Down
Loading

0 comments on commit e5f7642

Please sign in to comment.