diff --git a/.github/workflows/aws-auth.yml b/.github/workflows/aws-auth.yml new file mode 100644 index 00000000..d86468f0 --- /dev/null +++ b/.github/workflows/aws-auth.yml @@ -0,0 +1,64 @@ +name: Configure AWS Credentials + +on: + workflow_call: + inputs: + aws-region: + type: string + required: true + secrets: + role-to-assume: + required: true + gpg-passphrase: + required: true + outputs: + aws-access-key-id: + value: ${{ jobs.oidc-auth.outputs.aws-access-key-id }} + aws-secret-access-key: + value: ${{ jobs.oidc-auth.outputs.aws-secret-access-key }} + aws-session-token: + value: ${{ jobs.oidc-auth.outputs.aws-session-token }} + +permissions: + contents: read + id-token: write + +jobs: + oidc-auth: + name: OIDC Auth + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + outputs: + aws-access-key-id: ${{ steps.encrypt-aws-access-key-id.outputs.out }} + aws-secret-access-key: ${{ steps.encrypt-aws-secret-access-key.outputs.out }} + aws-session-token: ${{ steps.encrypt-aws-session-token.outputs.out }} + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: audit + - id: auth + uses: aws-actions/configure-aws-credentials@04b98b3f9e85f563fb061be8751a0352327246b0 # v3.0.1 + with: + aws-region: us-west-2 + role-to-assume: "${{ secrets.role-to-assume }}" + - id: encrypt-aws-access-key-id + run: | + encrypted=$(gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 -o - <(echo "$AWS_ACCESS_KEY_ID") | base64 -w0) + echo "out=$encrypted" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - id: encrypt-aws-secret-access-key + run: | + encrypted=$(gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 -o - <(echo "$AWS_SECRET_ACCESS_KEY") | base64 -w0) + echo "out=$encrypted" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - id: encrypt-aws-session-token + run: | + encrypted=$(gpg --batch --yes --passphrase "$GPG_PASSPHRASE" -c --cipher-algo AES256 -o - <(echo "$AWS_SESSION_TOKEN") | base64 -w0) + echo "out=$encrypted" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml deleted file mode 100644 index 94d824d8..00000000 --- a/.github/workflows/build-and-deploy.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Build and deploy - -on: - workflow_call: - inputs: - tf_backend_config_file: - type: string - required: true - tf_var_file: - type: string - required: true - secrets: - AWS_ROLE_TO_ASSUME: - required: true - DATADOG_API_KEY: - required: true - DATADOG_APP_KEY: - required: true - -concurrency: - group: ${{ github.workflow_ref }} - -permissions: - contents: read - id-token: write - -jobs: - deploy_terraform: - name: Deploy terraform - runs-on: ubuntu-latest - if: always() - env: - TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache - TF_VAR_version_identifier: ${{ github.sha }} - TF_VAR_git_commit_sha: ${{ github.sha }} - TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }} - TF_VAR_datadog_app_key: ${{ secrets.DATADOG_APP_KEY }} - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-taskfile - path: | - ~/.task - ~/bin - ~/build - ~/terraform/builds - - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - path: | - ~/.cache/go-build - ~/go/pkg/mod - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - - name: Install Taskfile - uses: arduino/setup-task@v1 - with: - version: 3.x - - name: Pre-build optimization - run: task prebuild-lambda - - name: Get project TF version - id: get_version - run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT - working-directory: terraform - - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: ${{ steps.get_version.outputs.TF_VERSION }} - - name: Ensure Terraform plugin cache exists - run: mkdir -p $TF_PLUGIN_CACHE_DIR - - name: Save/Restore Terraform plugin cache - uses: actions/cache@v3 - with: - path: ${{ env.TF_PLUGIN_CACHE_DIR }} - key: ${{ runner.os }}-terraform-${{ hashFiles('**/.terraform.lock.hcl') }} - restore-keys: | - ${{ runner.os }}-terraform- - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.AWS_ROLE_TO_ASSUME }}" - - name: Terraform Init - id: init - run: terraform init -backend-config="${{ inputs.tf_backend_config_file }}" - working-directory: terraform - - name: Terraform Validate - id: validate - run: terraform validate -no-color - working-directory: terraform - - name: Terraform Apply - if: steps.validate.outcome == 'success' - id: apply - run: terraform apply -auto-approve -input=false -no-color -var-file="${{ inputs.tf_var_file }}" - working-directory: terraform diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..36662694 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,322 @@ +name: Build + +on: + workflow_call: + inputs: + ref: + type: string + required: true + artifacts-retention-days: + description: Number of days to retain build artifacts + type: number + default: 90 + build-cli: + type: boolean + default: false + build-lambdas: + type: boolean + default: false + outputs: + build-cli-result: + value: ${{ jobs.build-cli.result }} + build-lambdas-result: + value: ${{ jobs.build-lambdas.result }} + cli-artifacts-key: + value: ${{ jobs.build-cli.outputs.artifacts-key }} + cli-artifacts-path: + value: ${{ jobs.build-cli.outputs.artifacts-path }} + cli-checksums-sha256: + value: ${{ jobs.build-cli.outputs.checksums-sha256 }} + lambda-artifacts-key: + value: ${{ jobs.build-lambdas.outputs.artifacts-key }} + lambda-artifacts-path: + value: ${{ jobs.build-lambdas.outputs.artifacts-path }} + lambda-checksums-sha256: + value: ${{ jobs.build-lambdas.outputs.checksums-sha256 }} + +jobs: + prepare: + runs-on: ubuntu-latest + env: + SOURCES_KEY: go-sources-${{ inputs.ref }} + SOURCES_PATH: | + ${{ github.workspace }}/cli + ${{ github.workspace }}/cmd + ${{ github.workspace }}/internal + ${{ github.workspace }}/pkg + ${{ github.workspace }}/openapi/openapi.yaml + ${{ github.workspace }}/go.mod + ${{ github.workspace }}/go.sum + ${{ github.workspace }}/Taskfile.yml + outputs: + sources-key: ${{ env.SOURCES_KEY }} + sources-path: ${{ env.SOURCES_PATH }} + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + sum.golang.org:443 + storage.googleapis.com:443 + - uses: actions/checkout@v4 + with: + show-progress: 'false' + persist-credentials: 'false' + - uses: actions/setup-go@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + go-version-file: go.mod + - uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + - name: Pre-build optimization + run: task prebuild-lambda + - name: Store build sources + id: store + uses: actions/upload-artifact@v3 + with: + name: ${{ env.SOURCES_KEY }} + path: ${{ env.SOURCES_PATH }} + if-no-files-found: error + retention-days: ${{ inputs.artifacts-retention-days }} + + build-lambdas: + name: Build Lambdas + if: needs.prepare.result == 'success' && inputs.build-lambdas + runs-on: ubuntu-latest + needs: + - prepare + env: + ARTIFACTS_KEY: lambdas-${{ inputs.ref }} + ARTIFACTS_PATH: ${{ github.workspace }}/bin + outputs: + artifacts-key: ${{ env.ARTIFACTS_KEY }} + artifacts-path: ${{ env.ARTIFACTS_PATH }} + checksums-sha256: ${{ steps.final-checksums.outputs.sha256 }} + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + sum.golang.org:443 + raw.githubusercontent.com:443 + - name: Restore Go build sources + uses: actions/download-artifact@v3 + with: + name: ${{ needs.prepare.outputs.sources-key }} + path: . + - uses: actions/setup-go@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + go-version-file: go.mod + - uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + - name: Prepare artifacts output directory + run: | + mkdir -p "$ARTIFACTS_PATH" + rm -rf "$ARTIFACTS_PATH/*" + - name: Build Lambdas + id: build + run: task build + - name: Get compiled checksums + id: compiled-checksums + run: | + COMPILED_CHECKSUMS=$(find "$ARTIFACTS_PATH" -type f -exec sha256sum -b {} \;) + echo "sha256<> $GITHUB_OUTPUT + echo "$COMPILED_CHECKSUMS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Install UPX + uses: crazy-max/ghaction-upx@0fc45e912669ba9e8fa2b430e97c8da2a632e29b # v3.0.0 + with: + version: v4.1.0 + install-only: true + - name: Run UPX + id: pack + run: | + UPX_RESULT=$(upx -5 -q "$ARTIFACTS_PATH"/*/bootstrap) + echo "result<> $GITHUB_OUTPUT + echo "$UPX_RESULT" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Get final checksums + id: final-checksums + run: | + FINAL_CHECKSUMS=$(find "$ARTIFACTS_PATH" -type f -exec sha256sum -b {} \;) + echo "sha256<> $GITHUB_OUTPUT + echo "$FINAL_CHECKSUMS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Publish build results + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Build Lambdas Summary + +
+ Compiled Checksums (before packing) + + ``` + ${{ env.COMPILED_CHECKSUMS }} + ``` + +
+
+ Final Checksums + + ``` + ${{ env.FINAL_CHECKSUMS }} + ``` + +
+
+ UPX Packing Results + + ``` + ${{ env.UPX_RESULT }} + ``` + +
+ ENDOFREPORT + cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY + env: + COMPILED_CHECKSUMS: ${{ steps.compiled-checksums.outputs.sha256 }} + FINAL_CHECKSUMS: ${{ steps.final-checksums.outputs.sha256 }} + UPX_RESULT: ${{ steps.pack.outputs.result }} + - name: Store build artifacts + id: store + uses: actions/upload-artifact@v3 + with: + name: ${{ env.ARTIFACTS_KEY }} + path: ${{ env.ARTIFACTS_PATH }} + if-no-files-found: error + retention-days: ${{ inputs.artifacts-retention-days }} + + build-cli: + name: Build CLI + if: needs.prepare.result == 'success' && inputs.build-cli + runs-on: ubuntu-latest + needs: + - prepare + env: + ARTIFACTS_KEY: cli-${{ inputs.ref }} + ARTIFACTS_PATH: ${{ github.workspace }}/bin/grants-ingest + outputs: + artifacts-key: ${{ env.ARTIFACTS_KEY }} + artifacts-path: ${{ env.ARTIFACTS_PATH }} + checksums-sha256: ${{ steps.final-checksums.outputs.sha256 }} + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + sum.golang.org:443 + raw.githubusercontent.com:443 + - name: Restore Go build sources + uses: actions/download-artifact@v3 + with: + name: ${{ needs.prepare.outputs.sources-key }} + path: . + - uses: actions/setup-go@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + go-version-file: go.mod + - uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + - name: Prepare artifacts output directory + run: | + mkdir -p $(dirname $ARTIFACTS_PATH) + rm "$ARTIFACTS_PATH" + - name: Build CLI + id: build + run: task build-cli + - name: Get compiled checksums + id: compiled-checksums + run: | + COMPILED_CHECKSUMS=$(sha256sum -b "$ARTIFACTS_PATH") + echo "sha256<> $GITHUB_OUTPUT + echo "$COMPILED_CHECKSUMS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Install UPX + uses: crazy-max/ghaction-upx@0fc45e912669ba9e8fa2b430e97c8da2a632e29b # v3.0.0 + with: + version: v4.1.0 + install-only: true + - name: Run UPX + id: pack + run: | + UPX_RESULT=$(upx -5 -q "$ARTIFACTS_PATH") + echo "result<> $GITHUB_OUTPUT + echo "$UPX_RESULT" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Get final checksums + id: final-checksums + run: | + FINAL_CHECKSUMS=$(sha256sum -b "$ARTIFACTS_PATH") + echo "sha256<> $GITHUB_OUTPUT + echo "$FINAL_CHECKSUMS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + - name: Publish build results + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Build Lambdas Summary + +
+ Compiled Checksums (before packing) + + ``` + ${{ env.COMPILED_CHECKSUMS }} + ``` + +
+
+ Final Checksums + + ``` + ${{ env.FINAL_CHECKSUMS }} + ``` + +
+
+ UPX Packing Results + + ``` + ${{ env.UPX_RESULT }} + ``` + +
+ ENDOFREPORT + cat "$REPORT_FILE" >> $GITHUB_STEP_SUMMARY + env: + COMPILED_CHECKSUMS: ${{ steps.compiled-checksums.outputs.sha256 }} + FINAL_CHECKSUMS: ${{ steps.final-checksums.outputs.sha256 }} + UPX_RESULT: ${{ steps.pack.outputs.result }} + - name: Store build artifacts + id: store + uses: actions/upload-artifact@v3 + with: + name: ${{ env.ARTIFACTS_KEY }} + path: ${{ env.ARTIFACTS_PATH }} + if-no-files-found: error + retention-days: ${{ inputs.artifacts-retention-days }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ecc8505e..023b9dff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,239 +1,83 @@ name: Continuous Integration on: - pull_request: {} - -permissions: - contents: read - pull-requests: write - id-token: write + pull_request_target: {} jobs: - dependency-review: - name: Dependency Review - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/dependency-review-action@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - qa: - name: QA - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Restore/save Taskfile cache - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-taskfile - path: | - ~/.task - ~/bin - ~/build - ~/cover.out - ~/cover.html - - uses: actions/setup-go@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - go-version-file: go.mod - - uses: arduino/setup-task@v1 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - version: 3.x - - name: Pre-build optimization - run: task prebuild-lambda - - name: Check Formatting - run: test -z "$(go fmt ./...)" || echo "Formatting check failed." - - name: Test - run: task test - - name: Vet - run: go vet ./... - - name: Lint - uses: dominikh/staticcheck-action@v1.3.0 - with: - install-go: false - - name: Build Lambdas - run: task build - - name: Build CLI - run: task build-cli - - tflint: - name: Lint terraform - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - name: Checkout source code - - uses: actions/cache@v3 - name: Cache plugin dir - with: - path: ~/.tflint.d/plugins - key: ${{ runner.os }}-tflint-${{ hashFiles('terraform/.tflint.hcl') }} - - uses: terraform-linters/setup-tflint@v3 - name: Setup TFLint - with: - tflint_version: latest - - name: Show TFLint version - run: tflint --version - - name: Init TFLint - run: tflint --init - working-directory: terraform - env: - GITHUB_TOKEN: ${{ github.token }} - - name: Run TFLint - run: tflint -f compact --recursive - - terraform_validate_plan_report: - name: Validate and plan terraform - runs-on: ubuntu-latest - if: always() - defaults: - run: - working-directory: terraform - env: - TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache - TF_VAR_version_identifier: ${{ github.sha }} - TF_VAR_git_commit_sha: ${{ github.sha }} - TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }} - TF_VAR_datadog_app_key: ${{ secrets.DATADOG_APP_KEY }} - concurrency: - group: run_terraform-staging - cancel-in-progress: false - steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-taskfile - path: | - ~/.task - ~/bin - ~/build - ~/terraform/builds - - uses: actions/cache@v3 - with: - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - path: | - ~/.cache/go-build - ~/go/pkg/mod - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version-file: go.mod - - name: Install Taskfile - uses: arduino/setup-task@v1 - with: - version: 3.x - - name: Pre-build optimization - run: task prebuild-lambda - - name: Get project TF version - id: get_version - run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT - - uses: hashicorp/setup-terraform@v2 - with: - terraform_version: ${{ steps.get_version.outputs.TF_VERSION }} - - name: Ensure Terraform plugin cache exists - run: mkdir -p $TF_PLUGIN_CACHE_DIR - - name: Save/Restore Terraform plugin cache - uses: actions/cache@v3 - with: - path: ${{ env.TF_PLUGIN_CACHE_DIR }} - key: ${{ runner.os }}-terraform-${{ hashFiles('**/.terraform.lock.hcl') }} - restore-keys: | - ${{ runner.os }}-terraform- - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.CI_ROLE_ARN }}" - - name: Terraform fmt - id: fmt - run: terraform fmt -check -diff -recursive - - name: Ensure Terraform plugin cache still exists - run: mkdir -p $TF_PLUGIN_CACHE_DIR - - name: Terraform Init - id: init - run: terraform init -backend-config="staging.s3.tfbackend" - - name: Terraform Validate - id: validate - run: terraform validate -no-color - - name: Terraform Plan - if: steps.validate.outcome == 'success' - id: plan - run: terraform plan -input=false -no-color -out=tfplan -var-file="staging.tfvars" && terraform show -no-color tfplan - - name: Reformat Plan - if: always() && steps.plan.outcome != 'cancelled' && steps.plan.outcome != 'skipped' - run: | - echo '${{ steps.plan.outputs.stdout || steps.plan.outputs.stderr }}' \ - | sed -E 's/^([[:space:]]+)([-+])/\2\1/g' > plan.txt - PLAN=$(cat plan.txt | head -c 65300) # Observe GitHub's 65535 character limit - echo "PLAN<> $GITHUB_ENV - echo "$PLAN" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Write the report markdown file - if: always() - run: | - REPORT_FILE=$(mktemp -t summary.md.XXXXX ) - echo "REPORT_FILE=$REPORT_FILE" >> $GITHUB_ENV - cat >> $REPORT_FILE << 'ENDOFREPORT' - ## Terraform Summary - - | Step | Result | - |:-----------------------------|:-------:| - | 🖌 Terraform Format & Style | ${{ (steps.fmt.outcome == 'success' && '✅') || (steps.fmt.outcome == 'skipped' && '➖') || '❌' }} | - | ⚙️ Terraform Initialization | ${{ (steps.init.outcome == 'success' && '✅') || (steps.init.outcome == 'skipped' && '➖') || '❌' }} | - | 🤖 Terraform Validation | ${{ (steps.validate.outcome == 'success' && '✅') || (steps.validate.outcome == 'skipped' && '➖') || '❌' }} | - | 📖 Terraform Plan | ${{ (steps.plan.outcome == 'success' && '✅') || (steps.plan.outcome == 'skipped' && '➖') || '❌' }} | - - ### Output - -
- Validation Output - - ``` - ${{ steps.validate.outputs.stdout }} - ``` - -
- -
- Plan Output - - ```diff - ${{ env.PLAN }} - ``` - -
- - *Pusher: @${{ github.actor }}, Action: `${{ github.event_name }}`, Workflow: [`${{ github.workflow }}`](${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }})* - ENDOFREPORT - - - name: Write the step summary - if: always() - run: | - cat $REPORT_FILE >> $GITHUB_STEP_SUMMARY - CONTENT=$(cat $REPORT_FILE) - echo "REPORT_CONTENT<> $GITHUB_ENV - echo "$CONTENT" >> $GITHUB_ENV - echo "ENDOFREPORT" >> $GITHUB_ENV - - name: Find previous report comment - if: always() - uses: peter-evans/find-comment@v2 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Terraform Summary - - name: Create or update comment - if: always() - uses: peter-evans/create-or-update-comment@v2 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body: ${{ env.REPORT_CONTENT }} - edit-mode: replace - - name: Print zip md5s - if: always() - run: md5sum builds/* - - name: Print bin md5s - if: always() - run: md5sum ../bin/*/* + permissions: + contents: read + uses: ./.github/workflows/qa.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + + build-lambdas: + permissions: + contents: read + name: Build Lambda handlers + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + build-cli: false + build-lambdas: true + artifacts-retention-days: 14 + + aws-auth: + name: Configure AWS Credentials + permissions: + contents: read + id-token: write + uses: ./.github/workflows/aws-auth.yml + with: + aws-region: us-west-2 + secrets: + role-to-assume: ${{ secrets.CI_ROLE_ARN }} + gpg-passphrase: ${{ secrets.TFPLAN_SECRET }} + + tf-plan: + name: Plan Terraform + permissions: + contents: read + needs: + - aws-auth + - build-lambdas + uses: ./.github/workflows/terraform-plan.yml + if: always() && needs.build-lambdas.outputs.build-lambdas-result == 'success' && needs.aws-auth.result == 'success' + with: + ref: ${{ github.event.pull_request.head.sha }} + concurrency-group: run_terraform-staging + bin-artifacts-key: ${{ needs.build-lambdas.outputs.lambda-artifacts-key }} + bin-artifacts-path: ${{ needs.build-lambdas.outputs.lambda-artifacts-path }} + aws-region: us-west-2 + environment-key: staging + tf-backend-config-file: staging.s3.tfbackend + tf-var-file: staging.tfvars + upload-artifacts: false + artifacts-retention-days: 14 + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.TFPLAN_SECRET }} + + publish-tf-plan: + name: Publish Terraform Plan + permissions: + contents: read + pull-requests: write + if: needs.tf-plan.result != 'skipped' || needs.tf-plan.result != 'cancelled' + needs: + - tf-plan + uses: ./.github/workflows/publish-terraform-plan.yml + with: + write-summary: true + write-comment: true + pr-number: ${{ github.event.pull_request.number }} + tf-fmt-outcome: ${{ needs.tf-plan.outputs.fmt-outcome }} + tf-init-outcome: ${{ needs.tf-plan.outputs.init-outcome }} + tf-plan-outcome: ${{ needs.tf-plan.outputs.plan-outcome }} + tf-plan-output: ${{ needs.tf-plan.outputs.plan-output }} + tf-validate-outcome: ${{ needs.tf-plan.outputs.validate-outcome }} + tf-validate-output: ${{ needs.tf-plan.outputs.validate-output }} diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml new file mode 100644 index 00000000..91794f72 --- /dev/null +++ b/.github/workflows/code-scanning.yml @@ -0,0 +1,57 @@ +name: "Code Scanning" + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: '35 8 * * 1-5' + +jobs: + dependency-review: + name: Dependency Review + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + - uses: actions/checkout@v4 + with: + show-progress: 'false' + persist-credentials: 'false' + - uses: actions/dependency-review-action@v3 + + codeql: + name: CodeQL + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: audit + - uses: actions/checkout@v4 + with: + show-progress: 'false' + persist-credentials: 'false' + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: go + queries: security-extended,security-and-quality + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:go" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 7083d746..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: - - main - pull_request: - branches: - - main - schedule: - - cron: '35 8 * * 1-5' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - steps: - - uses: actions/checkout@v3 - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: go - queries: security-extended,security-and-quality - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:go" diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 92b13eff..7a208731 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -6,7 +6,7 @@ on: - 'release/**' concurrency: - group: production + group: deploy-production cancel-in-progress: false permissions: @@ -14,144 +14,102 @@ permissions: id-token: write jobs: - plan: - name: Plan Deployment - runs-on: ubuntu-latest - defaults: - run: - working-directory: terraform - outputs: - terraform_plan_exitcode: ${{ steps.terraform_plan.outputs.exitcode }} - env: - TF_CLI_ARGS: "-no-color" - TF_INPUT: 0 - TF_VAR_version_identifier: ${{ github.ref_name }} - TF_VAR_git_commit_sha: ${{ github.sha }} - TF_VAR_datadog_api_key: ${{ secrets.DATADOG_API_KEY }} - TF_VAR_datadog_app_key: ${{ secrets.DATADOG_APP_KEY }} - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - token: ${{ secrets.GITHUB_TOKEN }} - go-version-file: go.mod - - name: Install Taskfile - uses: arduino/setup-task@v1 - with: - version: 3.x - - name: Pre-plan build optimizations - run: | - task prebuild-lambda - task build - - name: Get project TF version - id: get_version - run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT - working-directory: terraform - - uses: hashicorp/setup-terraform@v2 - with: - terraform_wrapper: true - terraform_version: ${{ steps.get_version.outputs.TF_VERSION }} - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.PRODUCTION_ROLE_ARN }}" - - name: Terraform Init - run: terraform init -backend-config="production.s3.tfbackend" - - name: Terraform Plan - id: terraform_plan - run: terraform plan -var-file="prod.tfvars" -out="tfplan" -detailed-exitcode - - name: Generate plaintext plan - id: show_plan - run: terraform show tfplan - - name: Reformat plan - run: | - echo '${{ steps.show_plan.outputs.stdout || steps.show_plan.outputs.stderr }}' \ - | sed -E 's/^([[:space:]]+)([-+])/\2\1/g' > plan.txt - PLAN=$(cat plan.txt | head -c 65300) - echo "PLAN<> $GITHUB_ENV - echo "$PLAN" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Write the step summary - run: | - REPORT_FILE=$(mktemp -t summary.md.XXXXX ) - echo "REPORT_FILE=$REPORT_FILE" >> $GITHUB_ENV - cat >> $REPORT_FILE << 'ENDOFREPORT' - ## Terraform Plan Result + build-lambdas: + name: Build Lambda handlers + permissions: + contents: read + uses: ./.github/workflows/build.yml + with: + ref: ${{ github.sha }} + build-cli: true + build-lambdas: true + artifacts-retention-days: 90 -
- Output + aws-auth: + name: Configure AWS Credentials + permissions: + contents: read + id-token: write + uses: ./.github/workflows/aws-auth.yml + with: + aws-region: us-west-2 + secrets: + gpg-passphrase: ${{ secrets.PRODUCTION_GPG_PASSPHRASE }} + role-to-assume: ${{ secrets.PRODUCTION_ROLE_ARN }} - ```diff - ${{ env.PLAN }} - ``` + tf-plan: + name: Plan Terraform + permissions: + contents: read + needs: + - aws-auth + - build-lambdas + uses: ./.github/workflows/terraform-plan.yml + with: + ref: ${{ github.sha }} + concurrency-group: run_terraform-production + bin-artifacts-key: ${{ needs.build-lambdas.outputs.lambda-artifacts-key }} + bin-artifacts-path: ${{ needs.build-lambdas.outputs.lambda-artifacts-path }} + aws-region: us-west-2 + environment-key: production + tf-backend-config-file: production.s3.tfbackend + tf-var-file: production.tfvars + upload-artifacts: true + artifacts-retention-days: 90 + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.PRODUCTION_GPG_PASSPHRASE }} -
- ENDOFREPORT - cat $REPORT_FILE >> $GITHUB_STEP_SUMMARY - - name: Encrypt terraform plan file - env: - PASSPHRASE: ${{ secrets.TFPLAN_SECRET }} - run: | - echo "$PASSPHRASE" | gpg --batch --yes --passphrase-fd 0 -c --cipher-algo AES256 tfplan - rm tfplan - - name: Store terraform artifacts - uses: actions/upload-artifact@v3 - with: - name: terraform-${{ github.sha }} - path: | - ${{ github.workspace }}/terraform - !${{ github.workspace }}/terraform/.terraform - - name: Store executable artifacts - uses: actions/upload-artifact@v3 - with: - name: bin-${{ github.sha }} - path: ${{ github.workspace }}/bin + publish-tf-plan: + name: Publish Terraform Plan + permissions: + contents: read + if: needs.tf-plan.result != 'skipped' || needs.tf-plan.result != 'cancelled' + needs: + - tf-plan + uses: ./.github/workflows/publish-terraform-plan.yml + with: + write-summary: true + write-comment: false + tf-fmt-outcome: ${{ needs.tf-plan.outputs.fmt-outcome }} + tf-init-outcome: ${{ needs.tf-plan.outputs.init-outcome }} + tf-plan-outcome: ${{ needs.tf-plan.outputs.plan-outcome }} + tf-plan-output: ${{ needs.tf-plan.outputs.plan-output }} + tf-validate-outcome: ${{ needs.tf-plan.outputs.validate-outcome }} + tf-validate-output: ${{ needs.tf-plan.outputs.validate-output }} - deploy: - name: Deploy to Production - runs-on: ubuntu-latest - environment: production + tf-apply: + name: Deploy to Staging needs: - - plan - if: always() && needs.plan.outputs.terraform_plan_exitcode == 2 - defaults: - run: - working-directory: terraform - env: - TF_CLI_ARGS: "-no-color" - TF_INPUT: 0 - steps: - - uses: hashicorp/setup-terraform@v2 - - name: Restore terraform artifacts - uses: actions/download-artifact@v3 - with: - name: terraform-${{ github.sha }} - path: ${{ github.workspace }}/terraform - - name: Restore executable artifacts - uses: actions/download-artifact@v3 - with: - name: bin-${{ github.sha }} - path: ${{ github.workspace }}/bin - - name: Decrypt terraform plan file - env: - GPG_PASSPHRASE: ${{ secrets.TFPLAN_SECRET }} - run: echo "$GPG_PASSPHRASE" | gpg -qd --batch --yes --passphrase-fd 0 -o tfplan tfplan.gpg - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-region: us-west-2 - role-to-assume: "${{ secrets.PRODUCTION_ROLE_ARN }}" - - name: Terraform Init - run: terraform init -backend-config="production.s3.tfbackend" - - name: Terraform Apply - run: terraform apply tfplan + - build-lambdas + - aws-auth + - tf-plan + if: needs.tf-plan.outputs.plan-exitcode == 2 + uses: ./.github/workflows/terraform-apply.yml + with: + bin-artifacts-key: ${{ needs.build-lambdas.outputs.lambda-artifacts-key }} + bin-artifacts-path: ${{ needs.build-lambdas.outputs.lambda-artifacts-path }} + tf-plan-artifacts-key: ${{ needs.tf-plan.outputs.artifacts-key }} + aws-region: us-west-2 + concurrency-group: run_terraform-production + tf-backend-config-file: production.s3.tfbackend + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.PRODUCTION_GPG_PASSPHRASE }} update_release: name: Update release runs-on: ubuntu-latest needs: - - deploy + - tf-apply env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index e0e138b2..2159aca4 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -6,7 +6,7 @@ on: - main concurrency: - group: run_terraform-staging + group: deploy-staging cancel-in-progress: false permissions: @@ -14,13 +14,93 @@ permissions: id-token: write jobs: - build_and_deploy: - name: Build and Deploy to Staging - uses: "./.github/workflows/build-and-deploy.yml" + build-lambdas: + name: Build Lambda handlers + permissions: + contents: read + uses: ./.github/workflows/build.yml with: - tf_backend_config_file: staging.s3.tfbackend - tf_var_file: staging.tfvars + ref: ${{ github.sha }} + build-cli: true + build-lambdas: true + artifacts-retention-days: 30 + + aws-auth: + name: Configure AWS Credentials + permissions: + contents: read + id-token: write + uses: ./.github/workflows/aws-auth.yml + with: + aws-region: us-west-2 + secrets: + gpg-passphrase: ${{ secrets.STAGING_GPG_PASSPHRASE }} + role-to-assume: ${{ secrets.STAGING_ROLE_ARN }} + + tf-plan: + name: Plan Terraform + permissions: + contents: read + needs: + - aws-auth + - build-lambdas + uses: ./.github/workflows/terraform-plan.yml + with: + ref: ${{ github.sha }} + concurrency-group: run_terraform-staging + bin-artifacts-key: ${{ needs.build-lambdas.outputs.lambda-artifacts-key }} + bin-artifacts-path: ${{ needs.build-lambdas.outputs.lambda-artifacts-path }} + aws-region: us-west-2 + environment-key: staging + tf-backend-config-file: staging.s3.tfbackend + tf-var-file: staging.tfvars + upload-artifacts: true + artifacts-retention-days: 30 + secrets: + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.STAGING_GPG_PASSPHRASE }} + + publish-tf-plan: + name: Publish Terraform Plan + permissions: + contents: read + if: needs.tf-plan.result != 'skipped' || needs.tf-plan.result != 'cancelled' + needs: + - tf-plan + uses: ./.github/workflows/publish-terraform-plan.yml + with: + write-summary: true + write-comment: false + tf-fmt-outcome: ${{ needs.tf-plan.outputs.fmt-outcome }} + tf-init-outcome: ${{ needs.tf-plan.outputs.init-outcome }} + tf-plan-outcome: ${{ needs.tf-plan.outputs.plan-outcome }} + tf-plan-output: ${{ needs.tf-plan.outputs.plan-output }} + tf-validate-outcome: ${{ needs.tf-plan.outputs.validate-outcome }} + tf-validate-output: ${{ needs.tf-plan.outputs.validate-output }} + + tf-apply: + name: Deploy to Staging + needs: + - build-lambdas + - aws-auth + - tf-plan + if: needs.tf-plan.outputs.plan-exitcode == 2 + uses: ./.github/workflows/terraform-apply.yml + with: + bin-artifacts-key: ${{ needs.build-lambdas.outputs.lambda-artifacts-key }} + bin-artifacts-path: ${{ needs.build-lambdas.outputs.lambda-artifacts-path }} + tf-plan-artifacts-key: ${{ needs.tf-plan.outputs.artifacts-key }} + aws-region: us-west-2 + concurrency-group: run_terraform-staging + tf-backend-config-file: staging.s3.tfbackend secrets: - AWS_ROLE_TO_ASSUME: "${{ secrets.STAGING_ROLE_ARN }}" - DATADOG_API_KEY: "${{ secrets.DATADOG_API_KEY }}" - DATADOG_APP_KEY: "${{ secrets.DATADOG_APP_KEY }}" + aws-access-key-id: ${{ needs.aws-auth.outputs.aws-access-key-id }} + aws-secret-access-key: ${{ needs.aws-auth.outputs.aws-secret-access-key }} + aws-session-token: ${{ needs.aws-auth.outputs.aws-session-token }} + datadog-api-key: ${{ secrets.DATADOG_API_KEY }} + datadog-app-key: ${{ secrets.DATADOG_APP_KEY }} + gpg-passphrase: ${{ secrets.STAGING_GPG_PASSPHRASE }} diff --git a/.github/workflows/publish-terraform-plan.yml b/.github/workflows/publish-terraform-plan.yml new file mode 100644 index 00000000..e9c0b7b9 --- /dev/null +++ b/.github/workflows/publish-terraform-plan.yml @@ -0,0 +1,137 @@ +name: Publish Terraform Plan + +on: + workflow_call: + inputs: + tf-fmt-outcome: + type: string + required: true + tf-init-outcome: + type: string + required: true + tf-plan-outcome: + type: string + required: true + tf-plan-output: + type: string + required: true + tf-validate-outcome: + type: string + required: true + tf-validate-output: + type: string + required: true + pr-number: + type: string + required: false + write-summary: + type: boolean + default: true + write-comment: + type: boolean + default: false + +permissions: + contents: read + pull-requests: write + +jobs: + publish: + name: Publish Terraform Plan + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + - name: Reformat Plan + run: | + PLAN=$(echo "$PLAN_RAW_OUTPUT" | sed -E 's/^([[:space:]]+)([-+])/\2\1/g') + echo "PLAN_REFORMATTED<> $GITHUB_ENV + echo "$PLAN" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + env: + PLAN_RAW_OUTPUT: ${{ inputs.tf-plan-output }} + - name: Write the report markdown file + run: | + REPORT_FILE=$(mktemp -t summary.md.XXXXX) + echo "REPORT_FILE=$REPORT_FILE" >> $GITHUB_ENV + cat >> $REPORT_FILE << 'ENDOFREPORT' + ## Terraform Summary + + | Step | Result | + |:-----------------------------|:-------:| + | 🖌 Terraform Format & Style | ${{ (env.TF_FMT_OUTCOME == 'success' && '✅') || (env.TF_FMT_OUTCOME == 'skipped' && '➖') || '❌' }} | + | ⚙️ Terraform Initialization | ${{ (env.TF_INIT_OUTCOME == 'success' && '✅') || (env.TF_INIT_OUTCOME == 'skipped' && '➖') || '❌' }} | + | 🤖 Terraform Validation | ${{ (env.TF_VALIDATE_OUTCOME == 'success' && '✅') || (env.TF_VALIDATE_OUTCOME == 'skipped' && '➖') || '❌' }} | + | 📖 Terraform Plan | ${{ (env.TF_PLAN_OUTCOME == 'success' && '✅') || (env.TF_PLAN_OUTCOME == 'skipped' && '➖') || '❌' }} | + + ### Output + +
+ Validation Output + + ``` + ${{ env.TF_VALIDATE_OUTPUT }} + ``` + +
+ +
+ Plan Output + + ```diff + ${{ env.TF_PLAN_OUTPUT }} + ``` + +
+ + *Pusher: @${{ env.GH_ACTOR }}, Action: `${{ env.GH_ACTION }}`, Workflow: [`${{ env.GH_WORKFLOW }}`](${{ env.GH_SERVER}}/${{ env.GH_REPO }}/actions/runs/${{ env.GH_RUN_ID }})* + ENDOFREPORT + env: + TF_FMT_OUTCOME: ${{ inputs.tf-fmt-outcome }} + TF_INIT_OUTCOME: ${{ inputs.tf-init-outcome }} + TF_VALIDATE_OUTCOME: ${{ inputs.tf-validate-outcome }} + TF_VALIDATE_OUTPUT: ${{ inputs.tf-validate-output }} + TF_PLAN_OUTCOME: ${{ inputs.tf-plan-outcome }} + TF_PLAN_OUTPUT: ${{ env.PLAN_REFORMATTED }} + GH_ACTOR: ${{ github.actor }} + GH_ACTION: ${{ github.event_name }} + GH_WORKFLOW: ${{ github.workflow }} + GH_SERVER: ${{ github.server_url }} + GH_REPO: ${{ github.repository }} + GH_RUN_ID: ${{ github.run_id }} + - name: Write the step summary + if: inputs.write-summary + run: cat $REPORT_FILE | head -c 65500 >> $GITHUB_STEP_SUMMARY # Observe GitHub's 65535 character limit + - name: Write the comment body + id: comment-body + run: | + CONTENT=$(cat $REPORT_FILE) + echo "REPORT_CONTENT<> $GITHUB_OUTPUT + echo "$CONTENT" >> $GITHUB_OUTPUT + echo "ENDOFREPORT" >> $GITHUB_OUTPUT + - name: Warn on missing comment requirements + if: inputs.write-comment && inputs.pr-number == '' + run: "echo 'WARNING: Cannot write a comment because pr-number is not set'" + - name: Find previous report comment + id: find-comment + if: inputs.write-comment && inputs.pr-number != '' + uses: peter-evans/find-comment@v2 + with: + issue-number: ${{ inputs.pr-number }} + comment-author: 'github-actions[bot]' + body-includes: Terraform Summary + - name: Create or update comment + if: always() + uses: peter-evans/create-or-update-comment@v2 + with: + comment-id: ${{ steps.find-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: ${{ steps.comment-body.outputs.REPORT_CONTENT }} + edit-mode: replace diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml new file mode 100644 index 00000000..3b606694 --- /dev/null +++ b/.github/workflows/qa.yml @@ -0,0 +1,102 @@ +name: QA Checks + +on: + workflow_call: + inputs: + ref: + type: string + required: true + +permissions: + contents: read + +jobs: + qa_go: + name: QA for Go + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + proxy.golang.org:443 + sum.golang.org:443 + storage.googleapis.com:443 + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Restore/save Taskfile cache + uses: actions/cache@v3 + with: + key: ${{ runner.os }}-qa-taskfile + path: | + ./.task + ./bin + ./cover.out + ./cover.html + - uses: actions/setup-go@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + go-version-file: go.mod + - uses: arduino/setup-task@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + version: 3.x + - name: Pre-build optimization + run: task prebuild-lambda + - name: Check Formatting + run: test -z "$(go fmt ./...)" || echo "Formatting check failed." + - name: Test + run: task test + - name: Vet + run: go vet ./... + - name: Lint + uses: dominikh/staticcheck-action@v1.3.0 + with: + install-go: false + - name: Ensure all go binaries compile + run: task build build-cli + + tflint: + name: Lint terraform + runs-on: ubuntu-latest + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + actions-results-receiver-production.githubapp.com:443 + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - uses: actions/cache@v3 + name: Cache plugin dir + with: + path: .tflint.d/plugins + key: ${{ runner.os }}-tflint-${{ hashFiles('terraform/.tflint.hcl') }} + - uses: terraform-linters/setup-tflint@v3 + name: Setup TFLint + with: + tflint_version: latest + - name: Show TFLint version + run: tflint --version + - name: Init TFLint + run: tflint --init + working-directory: terraform + env: + GITHUB_TOKEN: ${{ github.token }} + - name: Run TFLint + run: tflint -f compact --recursive diff --git a/.github/workflows/terraform-apply.yml b/.github/workflows/terraform-apply.yml new file mode 100644 index 00000000..55671a28 --- /dev/null +++ b/.github/workflows/terraform-apply.yml @@ -0,0 +1,131 @@ +name: Terraform Apply + +permissions: + contents: read + +on: + workflow_call: + inputs: + bin-artifacts-key: + type: string + required: true + bin-artifacts-path: + type: string + required: true + tf-plan-artifacts-key: + type: string + required: true + aws-region: + type: string + required: true + tf-backend-config-file: + type: string + required: true + concurrency-group: + description: Name of the concurrency group (avoids simultaneous Terraform execution against the same environment) + type: string + default: run_terraform + secrets: + aws-access-key-id: + required: true + aws-secret-access-key: + required: true + aws-session-token: + required: true + datadog-api-key: + required: true + datadog-app-key: + required: true + gpg-passphrase: + required: true + +jobs: + do: + name: Apply Terraform from Plan + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: terraform + env: + AWS_DEFAULT_REGION: ${{ inputs.aws-region }} + AWS_REGION: ${{ inputs.aws-region }} + TF_CLI_ARGS: "-no-color" + TF_IN_AUTOMATION: "true" + TF_INPUT: 0 + TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache + concurrency: + group: ${{ inputs.concurrency-group }} + cancel-in-progress: false + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + *.amazonaws.com:443 + actions-results-receiver-production.githubapp.com:443 + api.datadoghq.com:443 + checkpoint-api.hashicorp.com:443 + github.com:443 + objects.githubusercontent.com:443 + registry.terraform.io:443 + releases.hashicorp.com:443 + - name: Download Terraform artifacts + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.tf-plan-artifacts-key }} + path: ${{ github.workspace }}/terraform + - name: Get project TF version + id: get_tf_version + run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT + - uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 + with: + terraform_version: ${{ steps.get_tf_version.outputs.TF_VERSION }} + - name: Download Lambda handler artifacts + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.bin-artifacts-key }} + path: ${{ inputs.bin-artifacts-path }} + - name: Decrypt plan file + run: gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o tfplan tfplan.gpg + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - id: decrypt-aws-access-key-id + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-access-key-id }} + - id: decrypt-aws-secret-access-key + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-secret-access-key }} + - id: decrypt-aws-session-token + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-session-token }} + - name: Terraform Init + run: terraform init + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + TF_CLI_ARGS_init: "-backend-config=${{ inputs.tf-backend-config-file }}" + - name: Terraform Apply + run: terraform apply tfplan + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" diff --git a/.github/workflows/terraform-plan.yml b/.github/workflows/terraform-plan.yml new file mode 100644 index 00000000..7979854d --- /dev/null +++ b/.github/workflows/terraform-plan.yml @@ -0,0 +1,210 @@ +name: Terraform Plan + +permissions: + contents: read + +on: + workflow_call: + inputs: + ref: + type: string + required: true + bin-artifacts-key: + type: string + required: true + bin-artifacts-path: + type: string + required: true + environment-key: + type: string + required: true + aws-region: + type: string + required: true + tf-backend-config-file: + type: string + required: true + tf-var-file: + type: string + required: true + artifacts-retention-days: + description: Number of days to retain build artifacts + type: number + default: 90 + upload-artifacts: + type: boolean + default: false + concurrency-group: + description: Name of the concurrency group (avoids simultaneous Terraform execution against the same environment) + type: string + default: run_terraform + secrets: + aws-access-key-id: + required: true + aws-secret-access-key: + required: true + aws-session-token: + required: true + datadog-api-key: + required: true + datadog-app-key: + required: true + gpg-passphrase: + required: true + outputs: + artifacts-key: + value: ${{ jobs.do.outputs.artifacts-key }} + fmt-outcome: + value: ${{ jobs.do.outputs.fmt_outcome }} + init-outcome: + value: ${{ jobs.do.outputs.init_outcome }} + validate-outcome: + value: ${{ jobs.do.outputs.validate_outcome }} + validate-output: + value: ${{ jobs.do.outputs.validate_output }} + plan-exitcode: + value: ${{ jobs.do.outputs.plan_exitcode }} + plan-outcome: + value: ${{ jobs.do.outputs.plan_outcome }} + plan-output: + value: ${{ jobs.do.outputs.plan_output }} + +jobs: + do: + name: Validate and plan terraform + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: terraform + outputs: + artifacts-key: ${{ env.ARTIFACTS_KEY }} + fmt_outcome: ${{ steps.fmt.outcome }} + init_outcome: ${{ steps.init.outcome }} + validate_outcome: ${{ steps.validate.outcome }} + validate_output: ${{ steps.validate.outputs.stdout }} + plan_exitcode: ${{ steps.plan.outputs.exitcode }} + plan_outcome: ${{ steps.plan.outcome }} + plan_output: ${{ steps.show_plan.outputs.stdout || steps.show_plan.outputs.stderr }} + env: + ARTIFACTS_KEY: terraform-${{ inputs.environment-key }}-${{ inputs.ref }} + AWS_DEFAULT_REGION: ${{ inputs.aws-region }} + AWS_REGION: ${{ inputs.aws-region }} + TF_CLI_ARGS: "-no-color" + TF_IN_AUTOMATION: "true" + TF_INPUT: 0 + TF_PLUGIN_CACHE_DIR: ~/.terraform.d/plugin-cache + concurrency: + group: ${{ inputs.concurrency-group }} + cancel-in-progress: false + steps: + - uses: step-security/harden-runner@8ca2b8b2ece13480cda6dacd3511b49857a23c09 # v2.5.1 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + *.amazonaws.com:443 + actions-results-receiver-production.githubapp.com:443 + api.datadoghq.com:443 + checkpoint-api.hashicorp.com:443 + github.com:443 + objects.githubusercontent.com:443 + registry.terraform.io:443 + releases.hashicorp.com:443 + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + show-progress: 'false' + persist-credentials: 'false' + - name: Validate workflow configuration + if: inputs.upload-artifacts && (env.GPG_PASSPHRASE == '') + run: | + echo 'gpg-passphrase is required when upload-artifacts is true' + exit 1 + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + - name: Download Lambda handler artifacts + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.bin-artifacts-key }} + path: ${{ inputs.bin-artifacts-path }} + - name: Get project TF version + id: get_tf_version + run: echo "TF_VERSION=$(cat .terraform-version | tr -d '[:space:]')" | tee -a $GITHUB_OUTPUT + - uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3 + with: + terraform_version: ${{ steps.get_tf_version.outputs.TF_VERSION }} + - name: Terraform fmt + id: fmt + run: terraform fmt -check -diff -recursive + - id: decrypt-aws-access-key-id + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-access-key-id }} + - id: decrypt-aws-secret-access-key + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-secret-access-key }} + - id: decrypt-aws-session-token + run: | + decrypted=$(gpg -qd --batch --yes --passphrase "$GPG_PASSPHRASE" -o - <(echo "$VALUE" | base64 -d)) + echo "::add-mask::${decrypted}" + echo "out=${decrypted}" >> $GITHUB_OUTPUT + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + VALUE: ${{ secrets.aws-session-token }} + - name: Terraform Init + id: init + run: terraform init + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + TF_CLI_ARGS_init: "-backend-config=${{ inputs.tf-backend-config-file }}" + - name: Terraform Validate + id: validate + run: terraform validate -no-color + - name: Terraform Plan + if: always() && steps.validate.outcome == 'success' + id: plan + run: terraform plan -out="tfplan" -detailed-exitcode + env: + AWS_ACCESS_KEY_ID: "${{ steps.decrypt-aws-access-key-id.outputs.out }}" + AWS_SECRET_ACCESS_KEY: "${{ steps.decrypt-aws-secret-access-key.outputs.out }}" + AWS_SESSION_TOKEN: "${{ steps.decrypt-aws-session-token.outputs.out }}" + GPG_PASSPHRASE: "" # Just in case + TF_CLI_ARGS_plan: "-var-file=${{ inputs.tf-var-file }}" + TF_VAR_version_identifier: ${{ inputs.ref }} + TF_VAR_git_commit_sha: ${{ inputs.ref }} + TF_VAR_datadog_api_key: ${{ secrets.datadog-api-key }} + TF_VAR_datadog_app_key: ${{ secrets.datadog-app-key }} + - name: Generate plaintext plan + id: show_plan + run: terraform show tfplan + - name: Encrypt terraform plan file + id: encrypt_plan + if: success() && inputs.upload-artifacts + env: + GPG_PASSPHRASE: ${{ secrets.gpg-passphrase }} + run: | + gpg --batch --yes --passphrase "$GPG_PASSPHRASE" 0 -c --cipher-algo AES256 tfplan + rm tfplan + - name: Store terraform artifacts + if: success() && inputs.upload-artifacts + uses: actions/upload-artifact@v3 + with: + name: ${{ env.ARTIFACTS_KEY }} + path: | + ${{ github.workspace }}/terraform + !${{ github.workspace }}/terraform/.terraform + if-no-files-found: error + retention-days: ${{ inputs.artifacts-retention-days }} diff --git a/terraform/local.tfvars b/terraform/local.tfvars index 6289938f..9dd76ddd 100644 --- a/terraform/local.tfvars +++ b/terraform/local.tfvars @@ -5,6 +5,7 @@ permissions_boundary_policy_name = "" datadog_enabled = false datadog_dashboards_enabled = false datadog_lambda_extension_version = "38" +lambda_binaries_autobuild = true lambda_default_log_retention_in_days = 7 lambda_default_log_level = "DEBUG" eventbridge_scheduler_enabled = false diff --git a/terraform/main.tf b/terraform/main.tf index d159d1ac..3b178c9b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -478,6 +478,7 @@ module "DownloadGrantsGovDB" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -498,6 +499,7 @@ module "SplitGrantsGovXMLDB" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -517,6 +519,7 @@ module "ReceiveFFISEmail" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -544,6 +547,7 @@ module "EnqueueFFISDownload" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -568,6 +572,7 @@ module "PersistGrantsGovXMLDB" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -588,6 +593,7 @@ module "DownloadFFISSpreadsheet" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -612,6 +618,7 @@ module "SplitFFISSpreadsheet" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -637,6 +644,7 @@ module "PersistFFISData" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -661,6 +669,7 @@ module "ExtractGrantsGovDBToXML" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables @@ -684,6 +693,7 @@ module "PublishGrantEvents" { lambda_artifact_bucket = module.lambda_artifacts_bucket.bucket_id log_retention_in_days = var.lambda_default_log_retention_in_days log_level = var.lambda_default_log_level + lambda_autobuild = var.lambda_binaries_autobuild lambda_binaries_base_path = local.lambda_binaries_base_path lambda_arch = var.lambda_arch additional_environment_variables = local.lambda_environment_variables diff --git a/terraform/modules/DownloadFFISSpreadsheet/main.tf b/terraform/modules/DownloadFFISSpreadsheet/main.tf index c6510d61..dad86cce 100644 --- a/terraform/modules/DownloadFFISSpreadsheet/main.tf +++ b/terraform/modules/DownloadFFISSpreadsheet/main.tf @@ -54,6 +54,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/DownloadFFISSpreadsheet/variables.tf b/terraform/modules/DownloadFFISSpreadsheet/variables.tf index 1637764e..c04dbd12 100644 --- a/terraform/modules/DownloadFFISSpreadsheet/variables.tf +++ b/terraform/modules/DownloadFFISSpreadsheet/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/DownloadGrantsGovDB/main.tf b/terraform/modules/DownloadGrantsGovDB/main.tf index 8656eb86..eb8a766c 100644 --- a/terraform/modules/DownloadGrantsGovDB/main.tf +++ b/terraform/modules/DownloadGrantsGovDB/main.tf @@ -52,7 +52,9 @@ module "lambda_execution_policy" { } module "lambda_artifact" { - source = "../taskfile_lambda_builder" + source = "../taskfile_lambda_builder" + + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/DownloadGrantsGovDB/variables.tf b/terraform/modules/DownloadGrantsGovDB/variables.tf index fdd8da8d..53b87c69 100644 --- a/terraform/modules/DownloadGrantsGovDB/variables.tf +++ b/terraform/modules/DownloadGrantsGovDB/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/EnqueueFFISDownload/main.tf b/terraform/modules/EnqueueFFISDownload/main.tf index b67ee024..63a59594 100644 --- a/terraform/modules/EnqueueFFISDownload/main.tf +++ b/terraform/modules/EnqueueFFISDownload/main.tf @@ -54,6 +54,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/EnqueueFFISDownload/variables.tf b/terraform/modules/EnqueueFFISDownload/variables.tf index 68a30f49..075b0ad9 100644 --- a/terraform/modules/EnqueueFFISDownload/variables.tf +++ b/terraform/modules/EnqueueFFISDownload/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/ExtractGrantsGovDBToXML/main.tf b/terraform/modules/ExtractGrantsGovDBToXML/main.tf index 2d8deecb..93da204e 100644 --- a/terraform/modules/ExtractGrantsGovDBToXML/main.tf +++ b/terraform/modules/ExtractGrantsGovDBToXML/main.tf @@ -68,6 +68,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/ExtractGrantsGovDBToXML/variables.tf b/terraform/modules/ExtractGrantsGovDBToXML/variables.tf index e53c825c..8a1c52f9 100644 --- a/terraform/modules/ExtractGrantsGovDBToXML/variables.tf +++ b/terraform/modules/ExtractGrantsGovDBToXML/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/PersistFFISData/main.tf b/terraform/modules/PersistFFISData/main.tf index 9ffeaf86..5a681644 100644 --- a/terraform/modules/PersistFFISData/main.tf +++ b/terraform/modules/PersistFFISData/main.tf @@ -55,6 +55,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/PersistFFISData/variables.tf b/terraform/modules/PersistFFISData/variables.tf index af94bc83..012b7338 100644 --- a/terraform/modules/PersistFFISData/variables.tf +++ b/terraform/modules/PersistFFISData/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/PersistGrantsGovXMLDB/main.tf b/terraform/modules/PersistGrantsGovXMLDB/main.tf index 47afe596..97f63bab 100644 --- a/terraform/modules/PersistGrantsGovXMLDB/main.tf +++ b/terraform/modules/PersistGrantsGovXMLDB/main.tf @@ -55,6 +55,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/PersistGrantsGovXMLDB/variables.tf b/terraform/modules/PersistGrantsGovXMLDB/variables.tf index af94bc83..012b7338 100644 --- a/terraform/modules/PersistGrantsGovXMLDB/variables.tf +++ b/terraform/modules/PersistGrantsGovXMLDB/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/PublishGrantEvents/main.tf b/terraform/modules/PublishGrantEvents/main.tf index 61169ba1..02af75f9 100644 --- a/terraform/modules/PublishGrantEvents/main.tf +++ b/terraform/modules/PublishGrantEvents/main.tf @@ -41,6 +41,7 @@ data "aws_dynamodb_table" "source" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/PublishGrantEvents/variables.tf b/terraform/modules/PublishGrantEvents/variables.tf index 3c538113..fc0eba44 100644 --- a/terraform/modules/PublishGrantEvents/variables.tf +++ b/terraform/modules/PublishGrantEvents/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/ReceiveFFISEmail/main.tf b/terraform/modules/ReceiveFFISEmail/main.tf index 45af4b7a..c95a2288 100644 --- a/terraform/modules/ReceiveFFISEmail/main.tf +++ b/terraform/modules/ReceiveFFISEmail/main.tf @@ -64,6 +64,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/ReceiveFFISEmail/variables.tf b/terraform/modules/ReceiveFFISEmail/variables.tf index a10649a5..c6775de6 100644 --- a/terraform/modules/ReceiveFFISEmail/variables.tf +++ b/terraform/modules/ReceiveFFISEmail/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/SplitFFISSpreadsheet/main.tf b/terraform/modules/SplitFFISSpreadsheet/main.tf index 2a6db014..1d10cdd9 100644 --- a/terraform/modules/SplitFFISSpreadsheet/main.tf +++ b/terraform/modules/SplitFFISSpreadsheet/main.tf @@ -65,6 +65,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/SplitFFISSpreadsheet/variables.tf b/terraform/modules/SplitFFISSpreadsheet/variables.tf index 0f894226..16660def 100644 --- a/terraform/modules/SplitFFISSpreadsheet/variables.tf +++ b/terraform/modules/SplitFFISSpreadsheet/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/SplitGrantsGovXMLDB/main.tf b/terraform/modules/SplitGrantsGovXMLDB/main.tf index ada9e141..a05825ce 100644 --- a/terraform/modules/SplitGrantsGovXMLDB/main.tf +++ b/terraform/modules/SplitGrantsGovXMLDB/main.tf @@ -66,6 +66,7 @@ module "lambda_execution_policy" { module "lambda_artifact" { source = "../taskfile_lambda_builder" + autobuild = var.lambda_autobuild binary_base_path = var.lambda_binaries_base_path function_name = var.function_name s3_bucket = var.lambda_artifact_bucket diff --git a/terraform/modules/SplitGrantsGovXMLDB/variables.tf b/terraform/modules/SplitGrantsGovXMLDB/variables.tf index 0f894226..16660def 100644 --- a/terraform/modules/SplitGrantsGovXMLDB/variables.tf +++ b/terraform/modules/SplitGrantsGovXMLDB/variables.tf @@ -31,6 +31,11 @@ variable "lambda_binaries_base_path" { type = string } +variable "lambda_autobuild" { + description = "When true, a Lambda handler binary will be compiled when missing or outdated. When false, the compiled Lambda handler binary must already exist under `lambda_binaries_base_path`." + type = bool +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string diff --git a/terraform/modules/taskfile_lambda_builder/main.tf b/terraform/modules/taskfile_lambda_builder/main.tf index 9f95c9ee..91fa7819 100644 --- a/terraform/modules/taskfile_lambda_builder/main.tf +++ b/terraform/modules/taskfile_lambda_builder/main.tf @@ -17,6 +17,8 @@ locals { } data "external" "build_command" { + count = var.autobuild ? 1 : 0 + program = ["${path.module}/script.bash"] query = { task_command = local.task_command } } diff --git a/terraform/modules/taskfile_lambda_builder/variables.tf b/terraform/modules/taskfile_lambda_builder/variables.tf index f33e7fd9..86714795 100644 --- a/terraform/modules/taskfile_lambda_builder/variables.tf +++ b/terraform/modules/taskfile_lambda_builder/variables.tf @@ -31,6 +31,12 @@ variable "override_taskfile_command" { default = null } +variable "autobuild" { + description = "Whether to issue a Taskfile command to compile the Lambda handler binary when missing or outdated. When false, only a preexisting binary will be used. Recommendation: 'true' for development; 'false' for CI/CD." + type = bool + default = true +} + variable "override_path_to_binary" { description = "Explicit path to the file (outputted by the Taskfile command) that will be zipped. Uses '//bootstrap' by default." type = string diff --git a/terraform/production.tfvars b/terraform/production.tfvars index bdfcd2f2..24e934cf 100644 --- a/terraform/production.tfvars +++ b/terraform/production.tfvars @@ -1,6 +1,7 @@ namespace = "grants_ingest" environment = "production" ssm_deployment_parameters_path_prefix = "/grants_ingest/production/deploy-config" +lambda_binaries_autobuild = false lambda_default_log_retention_in_days = 30 lambda_default_log_level = "INFO" ffis_ingest_email_address = "ffis-ingest@grants.usdigitalresponse.org" diff --git a/terraform/staging.tfvars b/terraform/staging.tfvars index 0291b594..7b423897 100644 --- a/terraform/staging.tfvars +++ b/terraform/staging.tfvars @@ -2,6 +2,7 @@ namespace = "grants_ingest" environment = "staging" ssm_deployment_parameters_path_prefix = "/grants_ingest/deploy-config" datadog_enabled = true +lambda_binaries_autobuild = false lambda_default_log_retention_in_days = 30 lambda_default_log_level = "INFO" datadog_draft = true diff --git a/terraform/variables.tf b/terraform/variables.tf index f4afbfde..a761d1ce 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -61,6 +61,12 @@ variable "lambda_binaries_base_path" { default = "" } +variable "lambda_binaries_autobuild" { + description = "Whether to use Taskfile to compile missing/outdated Lambda handler binaries. Set to false during CI/CD." + type = bool + default = false +} + variable "lambda_arch" { description = "The target build architecture for Lambda functions (either x86_64 or arm64)." type = string