Merge pull request #507 from alan-turing-institute/DSG-2019-12-Deploy…

…ment

Deployment branch for December 2019 DSG

new_dsg_environment/azure-runbooks/dsg_build_instructions.md

@@ -15,7 +15,7 @@
 
 - #### Download a client VPN certificate for the Safe Haven Management VNet
 
-  - Navigate to the Safe Haven Management (SHM) KeyVault in the Safe Haven Management subscription via `Resource Groups -> RG_DSG_SECRETS -> dsg-management-<shm-id>`, where `<shm-id>` is `prod` for the production SHM environment and `test` for the test SHM environment.
+  - Navigate to the Safe Haven Management (SHM) KeyVault in the Safe Haven Management subscription via `Resource Groups -> RG_DSG_SECRETS -> kv-shm-<shm-id>`.
 
   - Once there open the "Certificates" page under the "Settings" section in the left hand sidebar.
 
@@ -33,7 +33,7 @@
 
   - Click the "Download VPN client" link at the top of the page to get the root certificate (VpnServerRoot.cer) and VPN configuration file (VpnSettings.xml), then follow the [VPN set up instructions](https://docs.microsoft.com/en-us/azure/vpn-gateway/point-to-site-vpn-client-configuration-azure-cert) using the Windows or Mac sections as appropriate.
 
-  - On Windows you may get a "|Windows protected your PC" pop up. If so, click `More info -> Run anyway`
+  - On Windows you may get a "Windows protected your PC" pop up. If so, click `More info -> Run anyway`
 
   - On Windows do not rename the vpn client as this will break it
 
@@ -123,7 +123,7 @@ The full configuration details for a new DSG are generated by defining a few "co
 
 ### Core SHM configuration properties
 The core properties for the relevant pre-existing Safe Haven Management (SHM) environment must be present in the `dsg_configs/core` folder.
-The following core SHM properties must be defined in a JSON file named `shm_<shm-id>_core_config.json`. See `shm_testc_core_config.json` for an example.
+The following core SHM properties must be defined in a JSON file named `shm_<shm-id>_core_config.json`.
 
 **NOTE:** The `netbiosName` fields must have a maximum length of 15 characters.
 
@@ -133,27 +133,16 @@ The following core SHM properties must be defined in a JSON file named `shm_<shm
     "computeVmImageSubscriptionName": "Azure Subscription name for compute VM",
     "domain": "The fully qualified domain name for the management environment",
     "netbiosname": "A short name to use as the local name for the domain. This must be 15 characters or less",
-    "shId": "A short ID to identify the management environment",
-    "name": "Deployment name",
+    "shmId": "A short ID to identify the management environment",
+    "name": "Safe Haven deployment name",
     "organisation": {
         "name": "Organisation name",
         "townCity": "Location",
         "stateCountyRegion": "Location",
         "countryCode": "e.g. GB"
     },
     "location": "The Azure location in which the management environment VMs are deployed",
-    "ipPrefix": "The three octet IP address prefix for the Class A range used by the management environment. Use 10.250.0",
-    "dcVmName":  "The VM name of the managment environment Active Directory Domain Controller",
-    "dcHostname":  "The hostname of the managment environment Active Directory Domain Controller",
-    "dcRgName": "The name of the Resource Group containing the managment environment Active Directory Domain Controller",
-    "vnetRgName":"The name of the Resource Group containing the Virtual Network for the management environment",
-    "npsIpLastOctet": "248",
-    "npsVmName": "The VM Name of the NPS VM",
-    "npsRgName": "The resources group containing the NPS VM",
-    "npsIp": "The IP address of the management environment NPS server",
-    "vnetName":"The name of the Virtual Network for the management environment",
-    "artifactStorageAccount": "The name of the storage account that will contain installation artifacts for new DSGs within the mangement  environment. Must be GLOBALLY unique within Azure. We suggest the format `dsg<shm-id>artifacts`",
-    "keyVaultName": "The name of the KeyVault that will contain secrets mangement environment. Must be GLOBALLY unique within Azure. We suggest the format `dsg-management-<shm-id>`"
+    "ipPrefix": "The three octet IP address prefix for the Class A range used by the management environment. Use 10.0.0 for this unless you have a good reason to use another prefix."
 }
 ```
 
@@ -217,6 +206,8 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Prepare SHM by running `./Prepare_SHM.ps1`, entering the DSG ID when prompted
 
+- This step also creates a DSG KeyVault in the DSG subscription in `Resource Groups -> RG_DSG_SECRETS -> kv-shm-<shm-id>-dsg<dsg-id>`. Additional deployment steps will add secrets to this KeyVault and you will need to access some of these for some of the manual configiration steps later.
+
 ## 2. Deploy Virtual Network
 
 ### Create the virtual network
@@ -235,7 +226,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 ### Set up a VPN connection to the DSG
 
-- In the **DSG subscription** open `Resource Groups -> RG_DSG_VNET -> DSG_VNET1_GW`
+- In the **DSG subscription** open `Resource Groups -> RG_DSG_VNET -> VNET_DSG<dsg-id>_GW`
 
   - Select "**Point to Site Configuration**" from the left-hand navigation
 
@@ -277,7 +268,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Connect to the new Domain controller via Remote Desktop client over the DSG VPN connection at the IP address `<dsg-identity-subnet-prefix>.250` (e.g. 10.250.x.250)
 
-- Login with local admin user `atiadmin` and the password for the DSG DC, which was created and stored in the `dsg<dsg-id>-dc-admin-password` secret in the Safe Haven Management KeyVault by the DC deployment script
+- Login with local admin user and password for the DSG DC, which were created and stored in the `dsg<dsg-id>-dc-admin-username` and `dsg<dsg-id>-dc-admin-password` secrets in the DSG KeyVault by the DC deployment script
 
 - From the "Server Management" application, select `Tools -> Group Policy Management`
 
@@ -331,7 +322,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Connect to the **SHM Domain Controller** via Remote Desktop client over the VPN connection
 
-- Login with domain user `<shm-domain>\atiadmin` and the SHM DC admin password from the `sh-management-dc-admin-password` secret in the Safe Haven Management KeyVault
+- Login with domain user `<shm-domain>\User` and the SHM DC admin password from the `shm-dc-admin-password` secret in the Safe Haven Management KeyVault
 
 - From the "Server Management" application, select `Tools -> Active Directory Domains and Trust`
 
@@ -349,7 +340,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
   | Trust Type:                                           | External Trust |
   | Direction of trust:                                   | Two-way |
   | Sides of trust:                                       | Both this domain and the specified domain |
-  |   User name and password:                               | Domain admin user on the DSG domain. Format: `<dsg-domain\Username>. User is "atiadmin ". See DSG DC admin secret in management KeyVault for password. |
+  |   User name and password:                               | Domain admin user on the DSG domain. Format: <dsg-domain>\Username>. See DSG `dsg<dsg-id>-dc-admin-username` and `dsg<dsg-id>-dc-admin-password` secrets in DSG KeyVault for username and password. |
   | Outgoing Trust Authentication Level-Local Domain:     | Domain-wide authentication  |
   | Outgoing Trust Authentication Level-Specified Domain: | Domain-wide authentication |
 
@@ -395,7 +386,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Connect to the **RDS Session Server 1 (RDSSH1)** via Remote Desktop client over the DSG VPN connection
 
-- Login with domain user `<dsg-domain>\atiadmin` and the **DSG DC** admin password from the `dsg<dsg-id>-dc-admin-password` secret from the SHM KeyVault (all DSG Windows servers use the same admin credentials)
+- Login with domain user `<dsg-domain>\Username`. See DSG `dsg<dsg-id>-dc-admin-username` and `dsg<dsg-id>-dc-admin-password` secrets in DSG KeyVault for username and password (all DSG Windows servers use the same admin credentials)
 
 - Open `C:\Software\rdssh1-app-server` in Windows explorer
 
@@ -407,7 +398,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Connect to the **RDS Gateway** via Remote Desktop client over the DSG VPN connection
 
-- Login with domain user `<dsg-domain>\atiadmin` and the **DSG DC** admin password from the `dsg<dsg-id>-dc-admin-password` secret from the SHM KeyVault (all DSG Windows servers use the same admin credentials)
+- Login with domain user `<dsg-domain>\Username`. See DSG `dsg<dsg-id>-dc-admin-username` and `dsg<dsg-id>-dc-admin-password` secrets in DSG KeyVault for username and password. (all DSG Windows servers use the same admin credentials)
 
 - Open a PowerShell command window with elevated privileges - make sure to use the `Windows PowerShell` application, not the `Windows PowerShell (x86)` application. The required server managment commandlets are not installed on the x86 version.
 
@@ -455,7 +446,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Enter the IP address of the NPS within the management domain (`10.251.0.248`)
 
-- Set the "Shared Secret" to the value of the `dsg-<dsg-id>-nps-secret` in the SHM KeyVault.
+- Set the "Shared Secret" to the value of the `dsg-<dsg-id>-nps-secret` in the DSG KeyVault.
 
   ![C:\\Users\\ROB\~1.CLA\\AppData\\Local\\Temp\\SNAGHTML2302f1a.PNG](images/media/image23.png)
 
@@ -523,7 +514,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Connect to the **SHM Domain Controller** via Remote Desktop client over the VPN connection
 
-- Login with domain user `<shm-domain>\atiadmin` and the SHM DC admin password from the `sh-management-dc-admin-password` secret in the Safe Haven Management KeyVault
+- Login with **SHM** domain user `<shm-domain>\User` See **SHM** `dsg<dsg-id>-dc-admin-username` and `shm-dc-admin-password` secrets in the **SHM** KeyVault for username and password.
 
 - In the "Server Management" app, click `Tools -> Active Directory Users and Computers`
 
@@ -557,7 +548,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
     - Ensure that the SHM NPS server RADIUS Client configuration is using the **private** IP address of the RDS Gateway and **not** its public one.
 
-    - Ensure the same shared secret from the `dsg-<dsg-id>-nps-secret` in the SHM KeyVault is used in **both** the SHM NPS server RADIUS Client configuration and the DSG RDS Gateway RD CAP Store configuration (see previous sections for instructions).
+    - Ensure the same shared secret from the `dsg-<dsg-id>-nps-secret` in the DSG KeyVault is used in **both** the SHM NPS server RADIUS Client configuration and the DSG RDS Gateway RD CAP Store configuration (see previous sections for instructions).
 
 - If you get a "We couldn't connect to the gateway because of an error" message, it's likely that the "Remote RADIUS Server" authentication timeouts have not been increased as described in a previous section. It seems that these are reset everytime the "Central CAP store" shared RADIUS secret is changed.
 
@@ -569,7 +560,7 @@ Each DSG must be assigned it's own unique IP address space, and it is very impor
 
 - Connect to the **RDS Session Server 2 (RDSSH1)** via Remote Desktop client over the DSG VPN connection
 
-- Login with domain user `<dsg-domain>\atiadmin` and the **DSG DC** admin password from the `dsg<dsg-id>-dc-admin-password` secret from the SHM KeyVault (all DSG Windows servers use the same admin credentials)
+- Login with domain user `<dsg-domain>\Username`. See DSG `dsg<dsg-id>-dc-admin-username` and `dsg<dsg-id>-dc-admin-password` secrets in DSG KeyVault for username and password (all DSG Windows servers use the same admin credentials)
 
 - Open `C:\Software\rdssh2-virtual-desktop-server` in Windows explorer
 
@@ -663,8 +654,8 @@ To deploy a compute VM you will need the following available on the machine you
 - Activate boot diagnostics on the VM and click save. You need to stay on that screen until the activation is complete.
 - Go back to the VM panel and click on the "Serial console" item near the bottom of the VM menu on the left habnd side of the VM panel.
 - If you are not prompted with `login:`, hit enter until the prompt appears
-- Enter `atiadmin` for the username
-- Enter the password from the `dsgroup<dsg-id>-dsvm-admin-password` secret in the `dsg-mangement-<shm-id>` KeyVault in the `RG_DSG_SECRETS` respource group of the SHM subscription.
+- Enter the username from the `dsg<dsg-id>-dsvm-admin-password` secret in the DSG KeyVault.
+- Enter the password from the `dsg<dsg-id>-dsvm-admin-password` secret in the DSG KeyVault.
 - To validate that our custom `cloud-init.yaml` file has been successfully uploaded, run `sudo cat /var/lib/cloud/instance/user-data.txt`. You should see the contents of the `new_dsg_environment/azure-vms/DSG_configs/cloud-init-compute-vm-DSG-<dsg-id>.yaml` file in the Safe Haven git repository.
 - To see the output of our custom `cloud-init.yaml` file, run `sudo tail -n 200 /var/log/cloud-init-output.log` and scroll up.
 
@@ -704,7 +695,7 @@ To run the smoke tests:
 
 - Connect to the **DSG Dataserver** via Remote Desktop client over the DSG VPN connection. Ensure that the Remote Desktop client configuration shares the Safe Haven repository folder on your local machine with the  Dataserver (or you have another way to transfer files between your local machine and the Dataserver VM).
 
-- Login with domain user `<dsg-domain>\atiadmin` and the **DSG DC** admin password from the SHM KeyVault (all DSG Windows servers use the same admin credentials)
+- Login with domain user `<dsg-domain>\Username`. See DSG `dsg<dsg-id>-dc-admin-username` and `dsg<dsg-id>-dc-admin-password` secrets in DSG KeyVault for username and password (all DSG Windows servers use the same admin credentials)
 
 - Copy the `package_lists` and `tests` folders from your local `<safe-haven-repository>/new_dsg_environment/azure-vms/` folder to a `dsg_tests` folder on within the `F:\Data` folder on the DSG Dataserver.
 

new_dsg_environment/azure-vms/configs/mirrors.sh

@@ -8,10 +8,10 @@ TIER="2"
 ADMIN_USERNAME="atiadmin"
 LOCATION="uksouth"
 MACHINENAME_BASE="Mirror"
-NSG_PREFIX="NSG_SHM_PKG_MIRRORS"
+NSG_PREFIX="NSG_SHM_TURING1_PKG_MIRRORS"
 SOURCEIMAGE="Canonical:UbuntuServer:18.04-LTS:latest"
 SUBNET_PREFIX="SBNT_SHM_PKG_MIRRORS"
-VNETNAME_PREFIX="VNET_SHM_PKG_MIRRORS"
+VNETNAME_PREFIX="VNET_SHM_TURING1_PKG_MIRRORS"
 
 # Disk sizes
 DATADISK_LARGE="8TB"

new_dsg_environment/dsg_configs/core/dsg_100_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE – Sandbox",
+    "adminSecurityGroupName" : "Safe Haven Production Admins",
+    "dsgId": "100",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup100.co.uk",
+    "netbiosName": "DSGROUP100",
+    "ipPrefix": "10.150.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_101_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 101 - Turkcell",
+    "adminSecurityGroupName" : "Safe Haven Production Admins",
+    "dsgId": "101",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup101.co.uk",
+    "netbiosName": "DSGROUP101",
+    "ipPrefix": "10.158.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_102_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 102 - Telus",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "102",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup102.co.uk",
+    "netbiosName": "DSGROUP102",
+    "ipPrefix": "10.166.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_103_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 103 - STC",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "103",
+    "shmId": "turing1",
+    "tier": "3",
+    "domain": "dsgroup103.co.uk",
+    "netbiosName": "DSGROUP103",
+    "ipPrefix": "10.172.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_104_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 104 - NATS",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "104",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup104.co.uk",
+    "netbiosName": "DSGROUP104",
+    "ipPrefix": "10.180.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_105_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE -105 - Roche",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "105",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup105.co.uk",
+    "netbiosName": "DSGROUP105",
+    "ipPrefix": "10.188.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_106_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 106 - Homeslink",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "106",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup106.co.uk",
+    "netbiosName": "DSGROUP106",
+    "ipPrefix": "10.196.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_107_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 107 - Ofstead",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "107",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup107.co.uk",
+    "netbiosName": "DSGROUP107",
+    "ipPrefix": "10.204.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_108_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 108 - WMCA",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "108",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup108.co.uk",
+    "netbiosName": "DSGROUP108",
+    "ipPrefix": "10.212.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_109_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 109 - Cochrane",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "109",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup109.co.uk",
+    "netbiosName": "DSGROUP109",
+    "ipPrefix": "10.220.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}

new_dsg_environment/dsg_configs/core/dsg_10_core_config.json

@@ -1,12 +1,14 @@
 {
-    "subscriptionName": "Data Study Group 10 (Prod)",
+    "subscriptionName": "Turing DSG - 10 - DST1LBField",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
     "dsgId": "10",
-    "shmId": "prod",
-    "tier": "2",
+    "shmId": "turing1",
+    "tier": "1",
     "domain": "dsgroup10.co.uk",
     "netbiosName": "DSGROUP10",
-    "ipPrefix": "10.250.72",
-    "rdsAllowedSources": "Internet",
+    "ipPrefix": "10.100.72",
+    "rdsAllowedSources": "default",
+    "rdsInternetAccess": "default",
     "computeVmImageType": "Ubuntu",
     "computeVmImageVersion": "0.1.2019082900"
-}
+}

new_dsg_environment/dsg_configs/core/dsg_110_core_config.json

@@ -0,0 +1,13 @@
+{
+    "subscriptionName": "Turing SRE - 110 - DNCP",
+    "adminSecurityGroupName": "Safe Haven Production Admins",
+    "dsgId": "110",
+    "shmId": "turing1",
+    "tier": "2",
+    "domain": "dsgroup110.co.uk",
+    "netbiosName": "DSGROUP110",
+    "ipPrefix": "10.228.0",
+    "rdsAllowedSources": "Internet",
+    "computeVmImageType": "Ubuntu",
+    "computeVmImageVersion": "0.1.2019082900"
+}