I want to deploy a shared service that has no Azure infrastructure, j…

…ust a connection URI Fixes microsoft#2485
marrobi · Oct 12, 2022 · 0795c54 · 0795c54
2 parents f36890f + 680d35a
commit 0795c54
Show file tree

Hide file tree

Showing 357 changed files with 8,538 additions and 2,455 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -24,7 +24,8 @@ ARG NODE_VERSION="lts/*"
 RUN su $USERNAME -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"
 
 # Install terraform
-ARG TERRAFORM_VERSION="1.2.7"
+# version 1.3.0/1 has issues with keep recreating certificate
+ARG TERRAFORM_VERSION="1.2.9"
 COPY .devcontainer/scripts/terraform.sh /tmp/
 RUN bash /tmp/terraform.sh "${TERRAFORM_VERSION}" /usr/bin
 
@@ -40,27 +41,27 @@ RUN apt-get update && apt-get install -y ca-certificates curl gnupg lsb-release
     && apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io --no-install-recommends \
     && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
-# Install nekos act - run GitHub workflows locally https://github.com/nektos/act
-RUN if [ "${INTERACTIVE}" = "true" ]; then curl https://raw.githubusercontent.com/nektos/act/master/install.sh | bash; fi
-
 # Install Certbot
-RUN apt-get update && apt-get install -y python3 python3-venv libaugeas0 --no-install-recommends \
+RUN if [ "${INTERACTIVE}" = "true" ]; then \
+    apt-get update && apt-get install -y python3 python3-venv libaugeas0 --no-install-recommends \
     && python3 -m venv /opt/certbot/ \
     && /opt/certbot/bin/pip install --no-cache-dir --upgrade pip \
     && /opt/certbot/bin/pip install --no-cache-dir certbot \
-    && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+    && apt-get clean -y && rm -rf /var/lib/apt/lists/* ; fi
 
 # Install Porter
 # Not using the script from https://cdn.porter.sh/latest/install-linux.sh
 # as it installs things we don't need and duplicates the binary.
 ARG PORTER_MIRROR=https://cdn.porter.sh
-ARG PORTER_PERMALINK=v0.38.12
+ARG PORTER_PERMALINK=v0.38.13
 ARG PORTER_PKG_PERMALINK=latest
+ARG PORTER_TERRAFORM_MIXIN_PKG_PERMALINK="v1.0.0-rc.1"
 ARG PORTER_HOME=/home/$USERNAME/.porter/
 COPY .devcontainer/scripts/porter.sh /tmp/
 RUN export PORTER_MIRROR=${PORTER_MIRROR} \
     PORTER_PERMALINK=${PORTER_PERMALINK} \
     PORTER_PKG_PERMALINK=${PORTER_PKG_PERMALINK} \
+    PORTER_TERRAFORM_MIXIN_PKG_PERMALINK=${PORTER_TERRAFORM_MIXIN_PKG_PERMALINK} \
     PORTER_HOME=${PORTER_HOME} \
     && /tmp/porter.sh
 

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -269,5 +269,5 @@
     8000
   ],
   // Give permission to access docker socket
-  "postCreateCommand": "sudo bash ./devops/scripts/set_docker_sock_permission.sh"
+  "postCreateCommand": "./.devcontainer/scripts/post-create.sh"
 }
diff --git a/.devcontainer/scripts/porter.sh b/.devcontainer/scripts/porter.sh
@@ -11,7 +11,7 @@ chmod +x "${PORTER_HOME}/porter"
 ln -s "${PORTER_HOME}/porter" "${PORTER_HOME}/runtimes/porter-runtime"
 
 "${PORTER_HOME}/porter" mixin install exec --version "${PORTER_PKG_PERMALINK}"
-"${PORTER_HOME}/porter" mixin install terraform --version "${PORTER_PKG_PERMALINK}"
+"${PORTER_HOME}/porter" mixin install terraform --version "${PORTER_TERRAFORM_MIXIN_PKG_PERMALINK}"
 "${PORTER_HOME}/porter" mixin install az --version "${PORTER_PKG_PERMALINK}"
 "${PORTER_HOME}/porter" plugin install azure --version "${PORTER_PKG_PERMALINK}"
 "${PORTER_HOME}/porter" mixin install docker --version "${PORTER_PKG_PERMALINK}"

diff --git a/.devcontainer/scripts/post-create.sh b/.devcontainer/scripts/post-create.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+# docker socket fixup
+sudo bash ./devops/scripts/set_docker_sock_permission.sh
+
+# install tre CLI
+(cd ./cli/ && make install-cli)  && echo -e "\n# Set up tre completion\nsource <(_TRE_COMPLETE=bash_source tre)" >> ~/.bashrc
+
diff --git a/.github/actions/devcontainer_run_command/action.yml b/.github/actions/devcontainer_run_command/action.yml
@@ -2,44 +2,14 @@
 name: "run_command"
 description: "Run a command in a devcontainer"
 inputs:
-  USE_ENV_VARS_NOT_FILES:
-    description: "Use ENV variables rather than a file."
-    required: false
-    default: "true"
-  TF_INPUT:
-    description: "Make Terraform fail if missing variables."
-    required: false
-    default: "0"
-  TF_IN_AUTOMATION:
-    description: "Terraform should not provide hints in the console."
-    required: false
-    default: "1"
   COMMAND:
     description: "The command you want to run in the Devcontainer."
     required: true
-  ACTIONS_ACR_NAME:
-    description: "The Azure Container registry name that the devcontainer is stored in."
-    required: true
-  ACTIONS_ACR_URI:
-    description: "The full uri of the Azure Container Registry where the devcontainer is stored in."
-    required: true
-  ACTIONS_ACR_PASSWORD:
-    description: "The Azure Container registry password for the devcontainer."
-    required: true
-  ACTIONS_DEVCONTAINER_TAG:
+  DEVCONTAINER_TAG:
     description: "The container label to use when running the command."
     required: true
-  ARM_TENANT_ID:
-    description: "Azure Tenant Id."
-    required: true
-  ARM_CLIENT_ID:
-    description: "Azure user Id. This user needs User Access Administrator permission as minimum."
-    required: true
-  ARM_CLIENT_SECRET:
-    description: "Azure user password."
-    required: true
-  ARM_SUBSCRIPTION_ID:
-    description: "Azure Subscription Id."
+  AZURE_CREDENTIALS:
+    description: "Credentials to access Azure."
     required: true
   API_CLIENT_ID:
     description: "The API Client Id."
@@ -65,34 +35,32 @@ inputs:
   TRE_ID:
     description: "The TRE Id."
     required: false
-  TF_VAR_terraform_state_container_name:
+  TERRAFORM_STATE_CONTAINER_NAME:
     description: "The name of the container to store the Terraform state."
     required: false
-  TF_VAR_mgmt_resource_group_name:
+    default: ""
+  MGMT_RESOURCE_GROUP_NAME:
     description: "The resource group used to store the Terraform state."
     required: false
-  TF_VAR_mgmt_storage_account_name:
+  MGMT_STORAGE_ACCOUNT_NAME:
     description: "The storage account used to store the Terraform state."
     required: false
-  TF_VAR_core_address_space:
+  CORE_ADDRESS_SPACE:
     description: "Core address space."
     required: false
-  TF_VAR_tre_address_space:
+  TRE_ADDRESS_SPACE:
     description: "TRE address apace."
     required: false
-  TF_VAR_swagger_ui_client_id:
+  SWAGGER_UI_CLIENT_ID:
     description: "The Swagger UI Client ID."
     required: false
-  TF_VAR_api_client_id:
-    description: "The API Client Id. (Same as Resource)"
-    required: false
-  TF_VAR_api_client_secret:
+  API_CLIENT_SECRET:
     description: "The API Client Secret."
     required: false
-  TF_VAR_application_admin_client_id:
+  APPLICATION_ADMIN_CLIENT_ID:
     description: "The Client ID of an identity that can manage the AAD Applications."
     required: false
-  TF_VAR_application_admin_client_secret:
+  APPLICATION_ADMIN_CLIENT_SECRET:
     description: "The Client secret of an identity that can manage the AAD Applications."
     required: false
   ACR_NAME:
@@ -111,111 +79,102 @@ inputs:
     description: "Indicates if the API endpoint has valid TLS certificate and if we validate it during E2E."
     required: false
     default: "true"
-  TF_VAR_stateful_resources_locked:
+  STATEFUL_RESOURCES_LOCKED:
     description: "A value indicating if resources with state will be protected with locks."
     required: false
     default: "true"
-  TF_VAR_enable_airlock_malware_scanning:
+  ENABLE_AIRLOCK_MALWARE_SCANNING:
     description: "If False, Airlock requests will skip the malware scanning stage."
     required: false
     default: "false"
   CI_CACHE_ACR_NAME:
     description: "A secondary ACR used for caching in CI environments"
-    required: false
+    required: true
     default: ""
   TF_LOG:
     description: "Log level for terraform - values are blank | DEBUG | TRACE"
     required: false
     default: ""
+  CORE_APP_SERVICE_PLAN_SKU:
+    description: "The AppService plan sku used in core resources"
+    required: false
+    default: ""
+  WORKSPACE_APP_SERVICE_PLAN_SKU:
+    description: "The AppService plan sku used by tests"
+    required: false
+    default: ""
 
 runs:
   using: composite
   steps:
-    - name: Login to Container Registry
-      uses: docker/login-action@v1
+    - name: Azure Login
+      uses: azure/login@v1
+      if: contains(inputs.COMMAND, 'bootstrap') != true
       with:
-        registry: ${{ inputs.ACTIONS_ACR_URI }}
-        username: ${{ inputs.ACTIONS_ACR_NAME }}
-        password: ${{ inputs.ACTIONS_ACR_PASSWORD }}
+        creds: ${{ inputs.AZURE_CREDENTIALS }}
+
+    - name: ACR Login
+      shell: bash
+      if: contains(inputs.COMMAND, 'bootstrap') != true
+      run: |
+        az acr login --name "${{ inputs.CI_CACHE_ACR_NAME }}"
 
     - name: Run command in DevContainer
       shell: bash
-      env:
-        ARM_TENANT_ID: "${{ inputs.ARM_TENANT_ID }}"
-        ARM_CLIENT_ID: "${{ inputs.ARM_CLIENT_ID }}"
-        ARM_CLIENT_SECRET: "${{ inputs.ARM_CLIENT_SECRET }}"
-        ARM_SUBSCRIPTION_ID: "${{ inputs.ARM_SUBSCRIPTION_ID }}"
-        API_CLIENT_ID: "${{ inputs.API_CLIENT_ID }}"
-        AAD_TENANT_ID: "${{ inputs.AAD_TENANT_ID }}"
-        TEST_WORKSPACE_APP_ID: "${{ inputs.TEST_WORKSPACE_APP_ID }}"
-        TEST_WORKSPACE_APP_SECRET: "${{ inputs.TEST_WORKSPACE_APP_SECRET }}"
-        TEST_APP_ID: "${{ inputs.TEST_APP_ID }}"
-        TEST_ACCOUNT_CLIENT_ID: "${{ inputs.TEST_ACCOUNT_CLIENT_ID }}"
-        TEST_ACCOUNT_CLIENT_SECRET: "${{ inputs.TEST_ACCOUNT_CLIENT_SECRET }}"
-        ACR_NAME: ${{ inputs.ACR_NAME }}
-        TRE_URL: "https://${{inputs.TRE_ID}}.${{inputs.LOCATION}}.cloudapp.azure.com"
-        TRE_ID: "${{ inputs.TRE_ID }}"
-        TF_VAR_tre_id: "${{ inputs.TRE_ID }}"
-        TF_VAR_terraform_state_container_name: "${{ inputs.TF_VAR_terraform_state_container_name }}"
-        TF_VAR_mgmt_resource_group_name: "${{ inputs.TF_VAR_mgmt_resource_group_name }}"
-        TF_VAR_mgmt_storage_account_name: "${{ inputs.TF_VAR_mgmt_storage_account_name }}"
-        TF_VAR_core_address_space: "${{ inputs.TF_VAR_core_address_space }}"
-        TF_VAR_tre_address_space: "${{ inputs.TF_VAR_tre_address_space }}"
-        TF_VAR_swagger_ui_client_id: "${{ inputs.TF_VAR_swagger_ui_client_id }}"
-        TF_VAR_aad_tenant_id: "${{ inputs.AAD_TENANT_ID }}"
-        TF_VAR_api_client_id: "${{ inputs.TF_VAR_api_client_id }}"
-        TF_VAR_api_client_secret: "${{ inputs.TF_VAR_api_client_secret }}"
-        TF_VAR_application_admin_client_id: "${{ inputs.TF_VAR_application_admin_client_id }}"
-        TF_VAR_application_admin_client_secret: "${{ inputs.TF_VAR_application_admin_client_secret }}"
-        TF_VAR_acr_name: ${{ inputs.ACR_NAME }}
-        IS_API_SECURED: ${{ inputs.IS_API_SECURED }}
       run: |
         docker run --rm --mount \
           "type=bind,src=${{ github.workspace }},dst=/workspaces/tre" \
           -v /var/run/docker.sock:/var/run/docker.sock \
           --workdir /workspaces/tre \
           --user vscode \
-          -e TF_INPUT="${{ inputs.TF_INPUT }}" \
-          -e TF_IN_AUTOMATION="${{ inputs.TF_IN_AUTOMATION }}" \
-          -e USE_ENV_VARS_NOT_FILES="${{ inputs.USE_ENV_VARS_NOT_FILES }}" \
+          -e TF_INPUT="0" \
+          -e TF_IN_AUTOMATION="1" \
+          -e USE_ENV_VARS_NOT_FILES="true" \
           -e BUNDLE_TYPE="${{ inputs.BUNDLE_TYPE }}" \
           -e WORKSPACE_SERVICE_NAME="${{ inputs.WORKSPACE_SERVICE_NAME }}" \
           -e LOCATION="${{ inputs.LOCATION }}" \
           -e TF_VAR_location="${{ inputs.LOCATION }}" \
           -e RESOURCE_LOCATION="${{ inputs.LOCATION }}" \
-          -e ARM_CLIENT_ID \
-          -e ARM_CLIENT_SECRET \
-          -e ARM_TENANT_ID \
-          -e ARM_SUBSCRIPTION_ID \
-          -e TF_VAR_terraform_state_container_name \
-          -e TF_VAR_mgmt_storage_account_name \
-          -e TF_VAR_mgmt_resource_group_name \
-          -e ACR_NAME \
-          -e TF_VAR_acr_name \
-          -e TF_VAR_aad_tenant_id \
-          -e TF_VAR_api_client_id \
-          -e TF_VAR_api_client_secret \
-          -e TF_VAR_application_admin_client_id \
-          -e TF_VAR_application_admin_client_secret \
-          -e TF_VAR_arm_subscription_id="${{ inputs.ARM_SUBSCRIPTION_ID }}" \
-          -e SWAGGER_UI_CLIENT_ID="${{ inputs.TF_VAR_swagger_ui_client_id }}" \
-          -e TF_VAR_swagger_ui_client_id \
-          -e TF_VAR_core_address_space \
-          -e TF_VAR_tre_address_space \
-          -e API_CLIENT_ID \
-          -e AAD_TENANT_ID \
-          -e TRE_ID \
-          -e TF_VAR_tre_id \
-          -e TRE_URL \
-          -e TEST_WORKSPACE_APP_ID \
-          -e TEST_WORKSPACE_APP_SECRET \
-          -e TEST_APP_ID \
-          -e TEST_ACCOUNT_CLIENT_ID \
-          -e TEST_ACCOUNT_CLIENT_SECRET \
-          -e IS_API_SECURED \
+          -e ARM_CLIENT_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).clientId }}" \
+          -e ARM_CLIENT_SECRET="${{ fromJSON(inputs.AZURE_CREDENTIALS).clientSecret }}" \
+          -e ARM_TENANT_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).tenantId }}" \
+          -e ARM_SUBSCRIPTION_ID="${{ fromJSON(inputs.AZURE_CREDENTIALS).subscriptionId }}" \
+          -e TF_VAR_terraform_state_container_name="${{ (inputs.TERRAFORM_STATE_CONTAINER_NAME != ''
+            && inputs.TERRAFORM_STATE_CONTAINER_NAME) || 'tfstate' }}" \
+          -e TF_VAR_mgmt_storage_account_name="${{ inputs.MGMT_STORAGE_ACCOUNT_NAME }}" \
+          -e TF_VAR_mgmt_resource_group_name="${{ inputs.MGMT_RESOURCE_GROUP_NAME }}" \
+          -e ACR_NAME=${{ inputs.ACR_NAME }} \
+          -e TF_VAR_acr_name=${{ inputs.ACR_NAME }} \
+          -e TF_VAR_aad_tenant_id="${{ inputs.AAD_TENANT_ID }}" \
+          -e TF_VAR_api_client_id="${{ inputs.API_CLIENT_ID }}" \
+          -e TF_VAR_api_client_secret="${{ inputs.API_CLIENT_SECRET }}" \
+          -e TF_VAR_application_admin_client_id="${{ inputs.APPLICATION_ADMIN_CLIENT_ID }}" \
+          -e TF_VAR_application_admin_client_secret="${{ inputs.APPLICATION_ADMIN_CLIENT_SECRET }}" \
+          -e TF_VAR_arm_subscription_id="${{ fromJSON(inputs.AZURE_CREDENTIALS).subscriptionId }}" \
+          -e SWAGGER_UI_CLIENT_ID="${{ inputs.SWAGGER_UI_CLIENT_ID }}" \
+          -e TF_VAR_swagger_ui_client_id="${{ inputs.SWAGGER_UI_CLIENT_ID }}" \
+          -e TF_VAR_core_address_space="${{ inputs.core_address_space }}" \
+          -e TF_VAR_tre_address_space="${{ inputs.tre_address_space }}" \
+          -e API_CLIENT_ID="${{ inputs.API_CLIENT_ID }}" \
+          -e AAD_TENANT_ID="${{ inputs.AAD_TENANT_ID }}" \
+          -e TRE_ID="${{ inputs.TRE_ID }}" \
+          -e TF_VAR_tre_id="${{ inputs.TRE_ID }}" \
+          -e TRE_URL="https://${{inputs.TRE_ID}}.${{inputs.LOCATION}}.cloudapp.azure.com" \
+          -e TEST_WORKSPACE_APP_ID="${{ inputs.TEST_WORKSPACE_APP_ID }}" \
+          -e TEST_WORKSPACE_APP_SECRET="${{ inputs.TEST_WORKSPACE_APP_SECRET }}" \
+          -e TEST_APP_ID="${{ inputs.TEST_APP_ID }}" \
+          -e TEST_ACCOUNT_CLIENT_ID="${{ inputs.TEST_ACCOUNT_CLIENT_ID }}" \
+          -e TEST_ACCOUNT_CLIENT_SECRET="${{ inputs.TEST_ACCOUNT_CLIENT_SECRET }}" \
+          -e IS_API_SECURED=${{ inputs.IS_API_SECURED }} \
           -e DOCKER_BUILDKIT=1 \
-          -e TF_VAR_stateful_resources_locked=${{ inputs.TF_VAR_stateful_resources_locked }} \
-          -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.TF_VAR_enable_airlock_malware_scanning }} \
+          -e TF_VAR_stateful_resources_locked=${{ inputs.STATEFUL_RESOURCES_LOCKED }} \
+          -e TF_VAR_enable_airlock_malware_scanning=${{ inputs.ENABLE_AIRLOCK_MALWARE_SCANNING }} \
           -e CI_CACHE_ACR_NAME="${{ inputs.CI_CACHE_ACR_NAME }}" \
-          '${{ inputs.ACTIONS_ACR_URI }}tredev:${{ inputs.ACTIONS_DEVCONTAINER_TAG }}' \
-        bash -c "${{ inputs.COMMAND }}"
+          -e TF_VAR_core_app_service_plan_sku="${{ (inputs.CORE_APP_SERVICE_PLAN_SKU != ''
+            && inputs.CORE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \
+          -e WORKSPACE_APP_SERVICE_PLAN_SKU="${{ (inputs.WORKSPACE_APP_SERVICE_PLAN_SKU != ''
+            && inputs.WORKSPACE_APP_SERVICE_PLAN_SKU) || 'P1v2' }}" \
+          '${{ inputs.CI_CACHE_ACR_NAME }}.azurecr.io/tredev:${{ inputs.DEVCONTAINER_TAG }}' \
+        bash -c "(cd cli/ && make install-cli) && ${{ inputs.COMMAND }}"
+        # Above command installs tre CLI (done via postCreateCommand in VS Code)
+        # If we switch to https://github.com/devcontainers/ci this would no longer be needed
diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml
@@ -14,8 +14,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.x
       - run: pip install -r docs/requirements.txt
-      - run: mkdocs gh-deploy --force
+      - run: mkdocs gh-deploy --strict --force
diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml
@@ -35,6 +35,8 @@ jobs:
               - 'templates/core/**/*'
             core_version:
               - 'templates/core/version.txt'
+            docs:
+              - 'docs/**/*'
 
       - name: Terraform format check
         if: ${{ steps.filter.outputs.terraform == 'true' }}
@@ -75,3 +77,9 @@ jobs:
           VALIDATE_DOCKERFILE_HADOLINT: true
           VALIDATE_TSX: true
           VALIDATE_TYPESCRIPT_ES: true
+
+      - name: Docs validation
+        if: ${{ steps.filter.outputs.docs == 'true' }}
+        run: |
+          pip install -r docs/requirements.txt
+          mkdocs build --strict