Merge pull request #545 from alan-turing-institute/DSG-2020-04-Deploy…

…ment

Changes from April 2020 DSG

deployment/administration/SHM_Shutdown_Deployment_VMs.ps1

@@ -7,15 +7,20 @@ Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SHM config
-$config = Get-ShmFullConfig($shmId)
 
-# Temporarily switch to SHM subscription
-$prevContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-ShmFullConfig $shmId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.subscriptionName
 
+
+# Stopping the deployment servers
+# -------------------------------
 Add-LogMessage -Level Info "Stopping all Deployment Servers"
 Get-AzVM -ResourceGroupName "RG_SHM_DEPLOYMENT_POOL" | Stop-AzVM -Force -NoWait
 
+
 # Switch back to original subscription
-$_ = Set-AzContext -Context $prevContext;
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/administration/SHM_Shutdown_IAAM_VMs.ps1

@@ -7,18 +7,23 @@ Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SHM config
-$config = Get-ShmFullConfig($shmId)
 
-# Temporarily switch to SHM subscription
-$prevContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-ShmFullConfig $shmId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.subscriptionName
 
+
+# Stop all IAAM VMs
+# -----------------
 Add-LogMessage -Level Info "Stopping NPS Server"
 Stop-AzVM -ResourceGroupName $config.nps.rg -Name $config.nps.vmName -Force -NoWait
 Add-LogMessage -Level Info "Stopping AD DCs"
 Stop-AzVM -ResourceGroupName $config.dc.rg -Name $config.dc.vmName -Force -NoWait
 Stop-AzVM -ResourceGroupName $config.dc.rg -Name $config.dcb.vmName -Force -NoWait
 
+
 # Switch back to original subscription
-$_ = Set-AzContext -Context $prevContext;
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/administration/SHM_Shutdown_Mirror_VMs.ps1

@@ -7,15 +7,19 @@ Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SHM config
-$config = Get-ShmFullConfig($shmId)
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-ShmFullConfig $shmId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.subscriptionName
 
-# Temporarily switch to SHM subscription
-$prevContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
 
-Add-LogMessage -Level Info "Stopping all Mirror Servers"
+# Stopping the package mirrors
+# ----------------------------
+Add-LogMessage -Level Info "Stopping all package mirror servers"
 Get-AzVM -ResourceGroupName "RG_SHM_PKG_MIRRORS" | Stop-AzVM -Force -NoWait
 
+
 # Switch back to original subscription
-$_ = Set-AzContext -Context $prevContext;
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/administration/SHM_Start_Deployment_VMs.ps1

@@ -5,17 +5,23 @@ param(
 
 Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
+Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SHM config
-$config = Get-ShmFullConfig($shmId)
 
-# Temporarily switch to SHM subscription
-$prevContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-ShmFullConfig $shmId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.subscriptionName
+
+
+# Start/restart the deployment servers
+# ------------------------------------
+Add-LogMessage -Level Info "Starting all Deployment Servers"
+Get-AzVM -ResourceGroupName "RG_SHM_DEPLOYMENT_POOL" | ForEach-Object { Enable-AzVM -Name $_.Name -ResourceGroupName $_.ResourceGroupName }
 
-Add-LogMessage -Level Info "Stopping all Deployment Servers"
-Get-AzVM -ResourceGroupName "RG_SHM_DEPLOYMENT_POOL"  | Restart-AzVM -NoWait
 
 # Switch back to original subscription
-$_ = Set-AzContext -Context $prevContext;
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/administration/SHM_Start_IAAM_VMs.ps1

@@ -5,22 +5,28 @@ param(
 
 Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
+Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SHM config
-$config = Get-ShmFullConfig($shmId)
 
-# Temporarily switch to SHM subscription
-$prevContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-ShmFullConfig $shmId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.subscriptionName
 
+
+# Start/restart all IAAM VMs
+# --------------------------
 Add-LogMessage -Level Info "Starting AD DCs..."
-Add-LogMessage -Level Info " - Waiting for Primary AD to start before starting other VMs."
-Restart-AzVM -ResourceGroupName $config.dc.rg -Name $config.dc.vmName
-Add-LogMessage -Level Info " - Waiting for Backup AD to start before starting other VMs."
-Restart-AzVM -ResourceGroupName $config.dc.rg -Name $config.dcb.vmName
-Add-LogMessage -Level Info "Starting NPS Server"
-Restart-AzVM -ResourceGroupName $config.nps.rg -Name $config.nps.vmName
+Add-LogMessage -Level Info "Starting Primary AD before other VMs..."
+Enable-AzVM -Name $config.dc.vmName -ResourceGroupName $config.dc.rg
+Add-LogMessage -Level Info "Starting Backup AD before other VMs..."
+Enable-AzVM -Name $config.dcb.vmName -ResourceGroupName $config.dc.rg
+Add-LogMessage -Level Info "Starting NPS Server..."
+Enable-AzVM -Name $config.nps.vmName -ResourceGroupName $config.nps.rg
+
 
 # Switch back to original subscription
-$_ = Set-AzContext -Context $prevContext;
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/administration/SHM_Start_Mirror_VMs.ps1

@@ -5,17 +5,23 @@ param(
 
 Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
+Import-Module $PSScriptRoot/../common/Deployments.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SHM config
-$config = Get-ShmFullConfig($shmId)
 
-# Temporarily switch to SHM subscription
-$prevContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.subscriptionName;
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-ShmFullConfig $shmId
+$originalContext = Get-AzContext
+$_ = Set-AzContext -SubscriptionId $config.subscriptionName
 
+
+# Start/restart the package mirrors
+# ---------------------------------
 Add-LogMessage -Level Info "Starting all Mirror Servers"
-Get-AzVM -ResourceGroupName "RG_SHM_PKG_MIRRORS" | Restart-AzVM -NoWait
+Get-AzVM -ResourceGroupName "RG_SHM_PKG_MIRRORS" | ForEach-Object { Enable-AzVM -Name $_.Name -ResourceGroupName $_.ResourceGroupName }
+
 
 # Switch back to original subscription
-$_ = Set-AzContext -Context $prevContext;
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext

deployment/administration/SRE_DSVM_Remote_Diagnostics.ps1

@@ -9,48 +9,99 @@ Import-Module Az
 Import-Module $PSScriptRoot/../common/Configuration.psm1 -Force
 Import-Module $PSScriptRoot/../common/Logging.psm1 -Force
 
-# Get SRE config
-# --------------
-$config = Get-SreConfig ($sreId);
+
+# Get config and original context before changing subscription
+# ------------------------------------------------------------
+$config = Get-SreConfig $sreId
 $originalContext = Get-AzContext
-$_ = Set-AzContext -SubscriptionId $config.sre.subscriptionName;
-
-
-# Find VM with private IP address matching the provided last octect
-## Turn provided last octect into full IP address in the data subnet
-$vmIpAddress = ($config.sre.network.subnets.data.prefix + "." + $ipLastOctet)
-Add-LogMessage -Level Info "Finding VM with IP ${vmIpAddress}..."
-## Get all compute VMs
-$computeVms = Get-AzVM -ResourceGroupName $config.sre.dsvm.rg
-## Get the NICs attached to all the compute VMs
-$computeVmNicIds = ($computeVms | ForEach-Object { (Get-AzVM -ResourceGroupName $config.sre.dsvm.rg -Name $_.Name).NetworkProfile.NetworkInterfaces.Id })
-$computeVmNics = ($computeVmNicIds | ForEach-Object { Get-AzNetworkInterface -ResourceGroupName $config.sre.dsvm.rg -Name $_.Split("/")[-1] })
-## Filter the NICs to the one matching the desired IP address and get the name of the VM it is attached to
-$computeVmName = ($computeVmNics | Where-Object { $_.IpConfigurations.PrivateIpAddress -match $vmIpAddress })[0].VirtualMachine.Id.Split("/")[-1]
-
-# Run remote scripts
-$diagnostic_scripts = @("check_ldap_connection.sh", "restart_name_resolution_service.sh", "rerun_realm_join.sh", "restart_sssd_service.sh")
-$testHost = $config.shm.dc.fqdn
-$ldapUser = $config.sre.users.ldap.dsvm.samAccountName
-$domainLower = $config.shm.domain.fqdn
-$servicePath = $config.shm.domain.serviceOuPath
+$_ = Set-AzContext -SubscriptionId $config.sre.subscriptionName
+
+
+# Find VM with private IP address matching the provided last octet
+# ----------------------------------------------------------------
+Add-LogMessage -Level Info "Finding compute VM with last IP octet: $ipLastOctet"
+$vmId = Get-AzNetworkInterface -ResourceGroupName $config.sre.dsvm.rg | Where-Object { ($_.IpConfigurations.PrivateIpAddress).Split(".") -eq $ipLastOctet } | ForEach-Object { $_.VirtualMachine.Id }
+$vm = Get-AzVM -ResourceGroupName $config.sre.dsvm.rg | Where-Object { $_.Id -eq $vmId }
+if ($?) {
+    Add-LogMessage -Level Success "Found compute VM '$($vm.Name)'"
+} else {
+    Add-LogMessage -Level Fatal "Could not find VM with last IP octet '$ipLastOctet'"
+}
 
+# Run remote diagnostic scripts
+# -----------------------------
+Add-LogMessage -Level Info "Running diagnostic scripts on VM $($vm.Name)..."
 $params = @{
     TEST_HOST = $config.shm.dc.fqdn
     LDAP_USER = $config.sre.users.ldap.dsvm.samAccountName
     DOMAIN_LOWER = $config.shm.domain.fqdn
-    SERVICE_PATH = "'" + $config.shm.domain.serviceOuPath + "'"
+    SERVICE_PATH = "'$($config.shm.domain.serviceOuPath)'"
+}
+foreach ($scriptNamePair in (("LDAP connection", "check_ldap_connection.sh"),
+                             ("name resolution", "restart_name_resolution_service.sh"),
+                             ("realm join", "rerun_realm_join.sh"),
+                             ("SSSD service", "restart_sssd_service.sh"))) {
+    $name, $diagnostic_script = $scriptNamePair
+    $scriptPath = Join-Path $PSScriptRoot ".." "secure_research_environment" "remote" "compute_vm" "scripts" $diagnostic_script
+    Add-LogMessage -Level Info "[ ] Configuring $name ($diagnostic_script) on compute VM '$($vm.Name)'"
+    $result = Invoke-RemoteScript -Shell "UnixShell" -ScriptPath $scriptPath -VMName $vm.Name -ResourceGroupName $config.sre.dsvm.rg -Parameter $params
+    $success = $?
+    Write-Output $result.Value
+    if ($success) {
+        Add-LogMessage -Level Success "Configuring $name on $($vm.Name) was successful"
+    } else {
+        Add-LogMessage -Level Failure "Configuring $name on $($vm.Name) failed!"
+    }
+}
+
+
+# Get LDAP secret from the KeyVault
+# ---------------------------------
+Add-LogMessage -Level Info "[ ] Loading LDAP secret from key vault '$($config.sre.keyVault.name)'"
+$kvLdapPassword = (Get-AzKeyVaultSecret -VaultName $config.sre.keyVault.Name -Name $config.sre.keyVault.secretNames.dsvmLdapPassword).SecretValueText;
+if ($kvLdapPassword) {
+    Add-LogMessage -Level Success "Found LDAP secret in the key vault"
+} else {
+    Add-LogMessage -Level Fatal "Could not load LDAP secret from key vault '$($config.sre.keyVault.name)'"
 }
 
-Add-LogMessage -Level Info "Running diagnostic scripts on VM ${computeVmName}..."
 
-foreach ($diagnostic_script in $diagnostic_scripts) {
-    $scriptPath = Join-Path $PSScriptRoot "remote_scripts" $diagnostic_script
-    $result = Invoke-AzVMRunCommand -ResourceGroupName $config.sre.dsvm.rg -Name "$computeVmName" `
-                                    -CommandId 'RunShellScript' -ScriptPath $scriptPath `
-                                    -Parameter $params
-    Write-Output $result.Value;
+# Set LDAP secret on the compute VM
+# ---------------------------------
+Add-LogMessage -Level Info "[ ] Setting LDAP secret on compute VM '$($vm.Name)'"
+$scriptPath = Join-Path $PSScriptRoot ".." "secure_research_environment" "remote" "compute_vm" "scripts" "reset_ldap_password.sh"
+$params = @{
+    ldapPassword = "`"$kvLdapPassword`""
+}
+$result = Invoke-RemoteScript -Shell "UnixShell" -ScriptPath $scriptPath -VMName $vm.Name -ResourceGroupName $config.sre.dsvm.rg -Parameter $params
+$success = $?
+Write-Output $result.Value
+if ($success) {
+    Add-LogMessage -Level Success "Setting LDAP secret on compute VM $($vm.Name) was successful"
+} else {
+    Add-LogMessage -Level Fatal "Setting LDAP secret on compute VM $($vm.Name) failed!"
+}
+
+
+# Set LDAP secret in local Active Directory on the SHM DC
+# -------------------------------------------------------
+$_ = Set-AzContext -SubscriptionId $config.shm.subscriptionName
+$scriptPath = Join-Path $PSScriptRoot ".." "secure_research_environment" "remote" "compute_vm" "scripts" "ResetLdapPasswordOnAD.ps1"
+$params = @{
+    samAccountName = "`"$($config.sre.users.ldap.dsvm.samAccountName)`""
+    ldapPassword = "`"$kvLdapPassword`""
 }
+Add-LogMessage -Level Info "[ ] Setting LDAP secret in local AD on '$($config.shm.dc.vmName)'"
+$result = Invoke-RemoteScript -Shell "PowerShell" -ScriptPath $scriptPath -VMName $config.shm.dc.vmName -ResourceGroupName $config.shm.dc.rg -Parameter $params
+$success = $?
+Write-Output $result.Value
+if ($success) {
+    Add-LogMessage -Level Success "Setting LDAP secret on SHM DC was successful"
+} else {
+    Add-LogMessage -Level Fatal "Setting LDAP secret on SHM DC failed!"
+}
+
 
-# Switch back to previous subscription
-$_ = Set-AzContext -Context $originalContext;
+# Switch back to original subscription
+# ------------------------------------
+$_ = Set-AzContext -Context $originalContext