build.yml

name: build

# Some words of caution before modifying this workflow:

# This file and workflow have been carefully architected to meet the following requirements:
#   * Builds and tests the correct artifacts in both CE and Ent while maintaining a merge-conflict-free
#     build.yml between the two repos
#   * Supports multiple Github event triggers
#   * Is highly optimized for cost and speed
#   * Supports a variety of complex use cases

# If you wish to modify this file/workflow, please consider:
#   * That the workflow must work under when triggered by pull_request, push, schedule, and
#     workflow_dispatch events.
#   * Merge-conflict-free compatibility between CE and Ent. Any changes that you make here must work
#     in both repository contexts.
#   * There are many workflow flow control modifiers. Further details below.
#   * The total number of workers and the runner size. Further details below.

# Further details:
#   * The workflow is used by the CRT system for building, notarizing, signing, and releasing
#     artifacts. Whatever we do in this workflow must support building all artifacts and uploading
#     them to Github in order to fulfill the CRT requirement, while also maintaining a smaller
#     default build matrix for the pull requests.
#   * CRT is designed to trigger a workflow called build in a workflow file called build.yml. This
#     file must build the correct artifacts in CE and Ent, depending on the repository context.
#     We've gone to great lengths to architect this file and workflow so that we can build and test
#     the correct artifacts in each context while maintaining a merge-conflict-free file between CE
#     and Ent. Any changes that you make here must work in both repository contexts.
#   * The workflow must support multiple event triggers, all of which have varying event payloads
#     which must be considered. If you make changes you must ensure that the workflow still works
#     under normal pull_request, push, schedule, and workflow_dispatch trigger events.
#   * The workflow has been highly optimized for cost and speed. If possible, it's better to add a
#     step to an existing job than create another job. Over a long time horizon a new job is often
#     much more expensive than a single step in an existing job, they also take up a limited number
#     of our available runners.
#   * Flow control in the workflow is complex in order to support many various use cases, including:
#     * Only building on tier 1 supported "core" artifacts by default.
#     * Only building the UI if the Go application or UI has been modified.
#     * Skipping builds entirely if the commit or PR only modifies changelog or website documentation.
#     * The ability to check out the HEAD reference instead of a Github merge branch reference.
#     * The ability to control building all of our tier 2 supported "extended" artifacts via a
#       build/all label, even if the event trigger is pull_request or, more importantly, a push.
#       It's important to note that we must maintain support for building all artifacts on push
#       via a pull request, even though push events aren't directly tied to pull requests. Our
#       label metadata helpers are designed to handle this complexity.
#     * The ability to build all of our artifacts on a scheduled cadence to ensure we don't
#       accidentally regress.
#     * All of these considerations, and many others, have led to the modular design we have here.
#     * If you're doing something in more than one place, try and use small composite actions
#       whenever possible.

on:
  workflow_dispatch:
  pull_request:
    types:
      - opened
      - ready_for_review
      - reopened
      - synchronize
  push:
    branches:
      - main
      - release/**
  schedule:
    - cron: '05 02 * * *' # * is a special character in YAML so you have to quote this string

concurrency:
  group: ${{ github.head_ref || github.run_id }}-build
  cancel-in-progress: true

jobs:
  setup:
    # Setup is our entrypoint into the entire workflow. Here we gather metadata and export useful
    # outputs for further use as inputs or for flow control.
    #
    # Trigger the setup workflow if any of the following conditions are true:
    #   * The workflow was triggered by a push (merge) to the main or release branch.
    #   * The workflow was triggered by pull request and the pull request is not a draft.
    #   * The workflow was triggered by on schedule to test building all artifacts.
    if: |
      github.event_name == 'push' ||
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
    runs-on: ${{ github.repository == 'hashicorp/vault' && 'ubuntu-latest' || fromJSON('["self-hosted","linux","small"]') }}
    outputs:
      app-changed: ${{ steps.changed-files.outputs.app-changed }}
      build-date: ${{ steps.metadata.outputs.vault-build-date }}
      checkout-ref: ${{ steps.checkout.outputs.ref }}
      compute-build: ${{ steps.metadata.outputs.compute-build }}
      compute-build-compat: ${{ steps.metadata.outputs.compute-build-compat }}
      compute-build-ui: ${{ steps.metadata.outputs.compute-build-ui }}
      compute-small: ${{ steps.metadata.outputs.compute-small }}
      docs-changed: ${{ steps.changed-files.outputs.docs-changed }}
      is-draft: ${{ steps.metadata.outputs.is-draft }}
      is-enterprise: ${{ steps.metadata.outputs.is-enterprise }}
      is-fork: ${{ steps.metadata.outputs.is-fork }}
      labels: ${{ steps.metadata.outputs.labels }}
      ui-changed: ${{ steps.changed-files.outputs.ui-changed }}
      vault-binary-name: ${{ steps.metadata.outputs.vault-binary-name }}
      vault-revision: ${{ steps.metadata.outputs.vault-revision }}
      vault-version: ${{ steps.metadata.outputs.vault-version }}
      vault-version-metadata: ${{ steps.metadata.outputs.vault-version-metadata }}
      vault-version-package: ${{ steps.metadata.outputs.vault-version-package }}
      workflow-trigger: ${{ steps.metadata.outputs.workflow-trigger }}
    steps:
      # Run the changed-files action to determine what Git reference we should check out
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - uses: ./.github/actions/changed-files
        id: changed-files
      - uses: ./.github/actions/checkout
        id: checkout # Make sure we check out correct ref after checking changed files
      # Get the vault version metadata
      - uses: hashicorp/actions-set-product-version@v2
        id: set-product-version
        with:
          checkout: false # don't override the reference we've checked out
      # Gather additional metadata about our execution context
      - uses: ./.github/actions/metadata
        id: metadata
        with:
          vault-version: ${{ steps.set-product-version.outputs.product-version }}
      - uses: ./.github/actions/set-up-go
        # Make sure all required Go modules are cached at this point. We don't want all of the Go
        # tests and build jobs to download modules and race to upload them to the cache.
        name: Ensure Go modules are cached
        with:
          github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }}
          # Don't download them on a cache hit during setup, just make sure they're cached before
          # subsequent workflows are run.
          no-restore: true

  ui:
    # The Web UI workflow is a prerequisite workflow for building our artifacts. If the application
    # or UI change we'll trigger this workflow but only build it if we don't already have the asset
    # in our Github cache.
    #
    # Ensure the Web UI is built if any of the following conditions are true:
    #   * The workflow was triggered by a push (merge) to the main or release branch.
    #   * The workflow was triggered by on schedule to test building all artifacts.
    #   * The `build/all` tag is present on either a pull request or on the pull request that created
    #     a merge
    #   * The workflow was triggered by a pull request, the pull request is not a draft, and the UI
    #     or app changed.
    if: |
      needs.setup.outputs.workflow-trigger == 'push' ||
      needs.setup.outputs.workflow-trigger == 'schedule' ||
      contains(fromJSON(needs.setup.outputs.labels), 'build/all') ||
      (
        needs.setup.outputs.workflow-trigger == 'pull_request' &&
        needs.setup.outputs.is-draft == 'false' &&
        (
          needs.setup.outputs.ui-changed == 'true' ||
          needs.setup.outputs.app-changed == 'true'
        )
      )
    needs: setup
    runs-on: ${{ fromJSON(needs.setup.outputs.compute-build-ui) }}
    outputs:
      cache-key: ui-${{ steps.ui-hash.outputs.ui-hash }}
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ needs.setup.outputs.checkout-ref }}
      - name: Get UI hash
        id: ui-hash
        run: echo "ui-hash=$(git ls-tree HEAD ui --object-only)" | tee -a "$GITHUB_OUTPUT"
      - name: Set up UI asset cache
        id: cache-ui-assets
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          enableCrossOsArchive: true
          lookup-only: true
          path: http/web_ui
          # Only restore the UI asset cache if we haven't modified anything in the ui directory.
          # Never do a partial restore of the web_ui if we don't get a cache hit.
          key: ui-${{ steps.ui-hash.outputs.ui-hash }}
      - if: steps.cache-ui-assets.outputs.cache-hit != 'true'
        name: Set up node and yarn
        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
        with:
          node-version-file: ui/package.json
          cache: yarn
          cache-dependency-path: ui/yarn.lock
      - if: steps.cache-ui-assets.outputs.cache-hit != 'true'
        name: Build UI
        run: make ci-build-ui

  artifacts:
    # Artifacts is where we'll build the various Vault binaries and package them into their respective
    # Zip bundles, RPM and Deb packages, and container images. After we've packaged them we upload
    # them to the Github Actions artifacts storage and execute our Enos test scenarios. If the
    # workflow is triggered by a push to main CRT will take these artifacts from Github and perform
    # all of the necessary notarizing and signing before uploading them to Artifactory.
    #
    #   # Trigger the setup workflow if any of the following conditions are true:
    #
    #   * The workflow was triggered by on schedule to test building all artifacts.
    #   * The Go app was changed.
    #   * The build/all label is present on a pull request or push.
    if: |
      needs.setup.outputs.workflow-trigger == 'schedule' ||
      needs.setup.outputs.app-changed == 'true' ||
      contains(fromJSON(needs.setup.outputs.labels), 'build/all')
    needs:
      - setup
      - ui # Don't build and test artifacts unless the UI build was triggered.
    # The following is the only line that should be different between CE and Ent.
    uses: ./.github/workflows/build-artifacts-ce.yml # Make sure we use the correct workflow.
    with:
      # The inputs defined here must be supported in both the build-artifacts-ce and
      # build-artifacts-ent workflows. The implementations should seek to keep a compatible interface.
      build-all: ${{ contains(fromJSON(needs.setup.outputs.labels), 'build/all') || needs.setup.outputs.workflow-trigger == 'schedule' }}
      build-date: ${{ needs.setup.outputs.build-date }}
      checkout-ref: ${{ needs.setup.outputs.checkout-ref }}
      compute-build: ${{ needs.setup.outputs.compute-build }}
      compute-build-compat: ${{ needs.setup.outputs.compute-build-compat }}
      compute-small: ${{ needs.setup.outputs.compute-small }}
      vault-revision: ${{ needs.setup.outputs.vault-revision }}
      vault-version: ${{ needs.setup.outputs.vault-version }}
      vault-version-package: ${{ needs.setup.outputs.vault-version-package }}
      web-ui-cache-key: ${{ needs.ui.outputs.cache-key }}
    secrets: inherit

  test:
    # Test all of the testable artifacts if our repo isn't a fork. We don't test when the PR is
    # created from a fork because secrets are not passed in and they are required.
    if: ${{ needs.setup.outputs.is-fork == 'false' }}
    name: test ${{ matrix.artifact }}
    needs:
      - setup
      - ui
      - artifacts
    uses: ./.github/workflows/test-run-enos-scenario-matrix.yml
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.artifacts.outputs.testable-packages) }}
    with:
      build-artifact-name: ${{ matrix.artifact }}
      runs-on: ${{ github.repository == 'hashicorp/vault' && '"ubuntu-latest"' || '["self-hosted","linux","small"]' }}
      sample-max: 1
      sample-name: ${{ matrix.sample }}
      ssh-key-name: ${{ github.event.repository.name }}-ci-ssh-key
      vault-edition: ${{ matrix.edition }}
      vault-revision: ${{ needs.setup.outputs.vault-revision }}
      vault-version: ${{ needs.setup.outputs.vault-version-metadata }}
    secrets: inherit

  test-containers:
    # Test all of the testable containers if our repo isn't a fork. We don't test when the PR is
    # created from a fork because secrets are not passed in and they are required (for now).
    if: ${{ needs.setup.outputs.is-fork == 'false' }}
    name: test ${{ matrix.artifact }}
    needs:
      - setup
      - ui
      - artifacts
    uses: ./.github/workflows/test-run-enos-scenario-containers.yml
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.artifacts.outputs.testable-containers) }}
    with:
      build-artifact-name: ${{ matrix.artifact }}
      sample-max: 1
      sample-name: ${{ matrix.sample }}
      vault-edition: ${{ matrix.edition }}
      vault-revision: ${{ needs.setup.outputs.vault-revision }}
      vault-version: ${{ needs.setup.outputs.vault-version-metadata }}
    secrets: inherit

  completed-successfully:
    # build/completed-successfully is the only build workflow that must pass in order to merge
    # a pull request. This workflow is used to determine the overall status of all the prior
    # workflows and to notify various different channels of success or failure. As part of this
    # workflow we create the necessary build metadata that is required for the CRT build system.
    #
    # Our logic here mirrors that of setup as it and this are the only two workflows that must
    # be triggered together.
    if: |
      always() &&
      (
        github.event_name == 'push' ||
        github.event_name == 'schedule' ||
        (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
      )
    runs-on: ${{ github.repository == 'hashicorp/vault' && 'ubuntu-latest' || fromJSON('["self-hosted","linux","small"]') }}
    permissions: write-all # Ensure we have id-token:write access for vault-auth.
    needs:
      - setup
      - ui
      - artifacts
      - test
      - test-containers
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - id: status
        name: Determine status
        run: |
          results=$(tr -d '\n' <<< '${{ toJSON(needs.*.result) }}')
          if ! grep -q -v -E '(failure|cancelled)' <<< "$results"; then
            result="failed"
          else
            result="success"
          fi
          {
            echo "result=${result}"
            echo "results=${results}"
          } | tee -a "$GITHUB_OUTPUT"
      - if: needs.setup.outputs.is-enterprise == 'true'
        id: vault-auth
        name: Vault Authenticate
        run: vault-auth
      - if: needs.setup.outputs.is-enterprise == 'true'
        id: secrets
        name: Fetch Vault Secrets
        uses: hashicorp/vault-action@v3
        with:
          url: ${{ steps.vault-auth.outputs.addr }}
          caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }}
          token: ${{ steps.vault-auth.outputs.token }}
          secrets: |
            kv/data/github/${{ github.repository }}/github_actions_notifications_bot token | SLACK_BOT_TOKEN;
      - id: slackbot-token
        run:
          echo "slackbot-token=${{ needs.setup.outputs.is-enterprise != 'true' && secrets.SLACK_BOT_TOKEN || steps.secrets.outputs.SLACK_BOT_TOKEN }}" >> "$GITHUB_OUTPUT"
      - if: |
          needs.setup.outputs.workflow-trigger == 'pull_request' &&
          github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name &&
          (github.repository == 'hashicorp/vault' || github.repository == 'hashicorp/vault-enterprise')
        name: Create or update a build status comment on the pull request
        env:
          ARTIFACTS: ${{ needs.artifacts.result }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          REPO: ${{ github.event.repository.name }}
          RUN_ID: ${{ github.run_id }}
          TEST: ${{ needs.test.result }}
          TEST_CONTAINERS: ${{ needs.test-containers.result }}
          UI: ${{ needs.ui.result }}
        run: ./.github/scripts/report-build-status.sh
      - name: Notify build failures in Slack
        if: |
          always() &&
          steps.status.outputs.result != 'success' &&
          (github.ref_name == 'main' || startsWith(github.ref_name, 'release/'))
        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e # v1.26.0
        env:
          SLACK_BOT_TOKEN: ${{ steps.slackbot-token.outputs.slackbot-token }}
        with:
          channel-id: "C05AABYEA9Y" # Notify #feed-vault-ci-official
          # channel-id: "C05Q4D5V89W" # Notify #test-vault-ci-slack-integration
          payload: |
            {
              "text": "${{ github.repository }} build failures on ${{ github.ref_name }}",
              "blocks": [
                {
                  "type": "header",
                  "text": {
                    "type": "plain_text",
                    "text": ":rotating_light: ${{ github.repository }} build failures on ${{ github.ref_name }} :rotating_light:",
                    "emoji": true
                  }
                },
                {
                  "type": "divider"
                },
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "${{ needs.setup.result != 'failure' && ':white_check_mark:' || ':x:' }} Setup\n${{ needs.ui.result != 'failure' && ':white_check_mark:' || ':x:' }} Build UI\n${{ needs.artifacts.result != 'failure' && ':white_check_mark:' || ':x:' }} Build Vault Artifacts\n${{ needs.test.result != 'failure' && ':white_check_mark:' || ':x:' }} Enos package test scenarios\n${{ needs.test-containers.result != 'failure' && ':white_check_mark:' || ':x:' }} Enos container test scenarios"
                  },
                  "accessory": {
                    "type": "button",
                    "text": {
                      "type": "plain_text",
                      "text": "View Failing Workflow",
                      "emoji": true
                    },
                    "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                  }
                }
              ]
            }
      - uses: hashicorp/actions-generate-metadata@v1
        if: needs.artifacts.result == 'success' # create build metadata if we successfully created artifacts
        id: generate-metadata-file
        with:
          version: ${{ needs.setup.outputs.vault-version-metadata }}
          product: ${{ needs.setup.outputs.vault-binary-name }}
      - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        if: steps.generate-metadata-file.outcome == 'success' # upload our metadata if we created it
        with:
          name: metadata.json
          path: ${{ steps.generate-metadata-file.outputs.filepath }}
          if-no-files-found: error
      - if: always() && steps.status.outputs.result != 'success'
        name: Check for failed status
        run: |
          echo "One or more required build workflows failed: ${{ steps.status.outputs.results }}"
          exit 1