update-dependencies-from-metadata.yml

name: Update Dependencies From Metadata (Retrieve, Metadata, Compile, Test, Create PR)

on:
  workflow_dispatch:
  schedule:
    - cron: '57 13 * * *' # daily at 13:57 UTC

jobs:
  retrieve:
    name: Retrieve New Versions and Generate Metadata
    runs-on: ubuntu-latest
    outputs:
      metadata-filepath: ${{ steps.retrieve.outputs.metadata-filepath }}
      metadata-json: ${{ steps.retrieve.outputs.metadata-json }}
      # from-source-metadata-filepath is the path to a file containing a subset
      # of metadata-json entries for NON-compiled dependencies
      from-source-metadata-filepath: ${{ steps.retrieve.outputs.from-source-metadata-filepath }}
      # compilation-json is a subset of metadata-json entries which are missing
      # a `checksum` and `uri`
      compilation-json: ${{ steps.retrieve.outputs.compilation-json }}
      id: ${{ steps.retrieve.outputs.id }}
      length: ${{ steps.retrieve.outputs.length }}
      compilation-length: ${{ steps.retrieve.outputs.compilation-length }}
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Setup Go
        uses: actions/setup-go@v5
        with:
          go-version: 'stable'

      - name: Run Retrieve
        id: retrieve
        working-directory: dependency
        run: |
          #!/usr/bin/env bash
          set -euo pipefail
          shopt -s inherit_errexit

          OUTPUT="/tmp/metadata.json"

          make retrieve \
            buildpackTomlPath="${{ github.workspace }}/buildpack.toml" \
            output="${OUTPUT}"

          id=$(jq -r .[0].id < "${OUTPUT}")
          content=$(jq -r < "${OUTPUT}")

          length=$(echo $content | jq -r '. | length')

          compilation=$(echo $content | jq -r 'map(select(.checksum == null and .uri == null))'?)
          complength=$(echo $compilation | jq -r '. | length')
          echo $content | jq -r 'map(select(.checksum != null and .uri != null))'? > "/tmp/from-source-metadata.json"
          echo "from-source-metadata-filepath=/tmp/from-source-metadata.json" >> "$GITHUB_OUTPUT"


          delimiter="$(uuidgen)"
          echo "metadata-filepath=${OUTPUT}" >> "$GITHUB_OUTPUT"
          printf "metadata-json<<%s\n%s\n%s\n" "${delimiter}" "${content}" "${delimiter}" >> "$GITHUB_OUTPUT" # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
          echo "id=$id" >> "$GITHUB_OUTPUT"
          echo "length=$length" >> "$GITHUB_OUTPUT"
          printf "compilation-json<<%s\n%s\n%s\n" "${delimiter}" "${compilation}" "${delimiter}" >> "$GITHUB_OUTPUT" # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
          echo "compilation-length=$complength" >> "$GITHUB_OUTPUT"


      - name: Upload `${{ steps.retrieve.outputs.metadata-filepath }}`
        uses: actions/upload-artifact@v4
        with:
          name: metadata.json
          path: ${{ steps.retrieve.outputs.metadata-filepath }}

      - name: Upload `${{ steps.retrieve.outputs.from-source-metadata-filepath }}`
        uses: actions/upload-artifact@v4
        with:
          name: from-source-metadata.json
          path: ${{ steps.retrieve.outputs.from-source-metadata-filepath }}

  # Check if there is buildpack-provided compilation code and testing code
  # Optional compilation code expected at: <buildpack>/dependency/actions/compile/
  # Optional testing code expected at: <buildpack>/dependency/test/
  get-compile-and-test:
    name: Get Compilation and Testing Code
    outputs:
      should-compile: ${{ steps.compile-check.outputs.should-compile }}
      should-test: ${{ steps.test-check.outputs.should-test }}
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Has Compilation Action?
        id: compile-check
        run: |
          if test -d "dependency/actions/compile"; then
            echo "Compilation action provided"
            echo "should-compile=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Has Testing Action?
        id: test-check
        run: |
          if test -d "dependency/test"; then
            echo "Testing file provided"
            echo "should-test=true" >> "$GITHUB_OUTPUT"
          fi

  test:
    name: Test Non-Compiled Dependency
    needs:
      - retrieve
      - get-compile-and-test
    strategy:
      matrix:
        includes: ${{ fromJSON(needs.retrieve.outputs.metadata-json) }}
    # Run job step if BOTH:
    #   (1) needs.get-compile-and-test.outputs.should-test = TRUE -> if there is a dependency/test directory in the buildpack
    #   (2) needs.get-compile-and-test.outputs.should-compile = FALSE -> if there is NOT a dependency/actions/compile directory in the buildpack
    #   AND:
    #   (3) there is at least one new version to test
    if: ${{ needs.retrieve.outputs.length > 0 && needs.get-compile-and-test.outputs.should-test == 'true' && needs.get-compile-and-test.outputs.should-compile == 'false' }}
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Make Temporary Artifact Directory
        id: make-outputdir
        run: echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT"

      # Download the tarball for testing if:
      #   (1) dependency testing code is present in the buildpack directory
      #   (2) URI in metadata.json is available
      - name: Download upstream tarball (if not compiled)
        if: ${{ matrix.includes.uri != ''  && needs.get-compile-and-test.outputs.should-test == 'true' }}
        run: |
          #!/usr/bin/env bash
          set -euo pipefail
          shopt -s inherit_errexit

          curl ${{ matrix.includes.uri }} \
            --fail-with-body \
            --show-error \
            --silent \
            --location \
            --output ${{ steps.make-outputdir.outputs.outputdir }}/dependency.tgz

      # Test the dependency tarball if:
      #   (1) dependency testing code is present in the buildpack directory
      - name: Test Upstream Dependency
        working-directory: dependency
        if: ${{ needs.get-compile-and-test.outputs.should-test == 'true' }}
        run: |
          make test \
            version="${{ matrix.includes.version }}" \
            tarballPath="${{ steps.make-outputdir.outputs.outputdir }}/*.tgz"
  compile:
    name: Compile and Test Dependency
    needs:
      - retrieve
      - get-compile-and-test
    strategy:
      matrix:
        includes: ${{ fromJSON(needs.retrieve.outputs.compilation-json) }}
    # Run job step if:
    #   (1) needs.get-compile-and-test.outputs.should-compile -> if there is a dependency/actions/compile directory in the buildpack
    #   (2) OR needs.get-compile-and-test.outputs.should-test -> if there is a dependency/test directory in the buildpack
    #   AND:
    #   (3) there is at least one version to compile/test
    if: ${{ needs.retrieve.outputs.compilation-length > 0 && (needs.get-compile-and-test.outputs.should-compile == 'true' || needs.get-compile-and-test.outputs.should-test == 'true') }}
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Make Temporary Artifact Directory
        id: make-outputdir
        run: |
          echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT"

      # Compile if all of the following conditions are met:
      #   (1) compilation Github Action presetn in the buildpack directory
      #   (2) checksum in metadata.json is empty
      #   (3) URI in metadata.json is empty
      - name: Compile version ${{ matrix.includes.version }} on ${{ matrix.includes.target }} Dockerfile
        id: compile
        if: ${{ needs.get-compile-and-test.outputs.should-compile && matrix.includes.checksum == '' && matrix.includes.uri == '' }}
        uses: ./dependency/actions/compile
        with:
          version: "${{ matrix.includes.version }}"
          outputdir: "${{ steps.make-outputdir.outputs.outputdir }}"
          target: "${{ matrix.includes.target }}"

      # If compiled, upload the tarball and checksum file for usage in the Update metadata job
      - name: Upload workflow asset
        uses: actions/upload-artifact@v4
        if: ${{ needs.get-compile-and-test.outputs.should-compile && matrix.includes.checksum == '' && matrix.includes.uri == '' }}
        with:
          name: '${{ needs.retrieve.outputs.id }}-${{ matrix.includes.version }}-${{ matrix.includes.target }}'
          path: '${{ steps.make-outputdir.outputs.outputdir }}/*'

      # Test the dependency tarball if:
      #   (1) dependency testing code is present in the buildpack directory
      - name: Test Dependency
        working-directory: dependency
        if: ${{ needs.get-compile-and-test.outputs.should-test == 'true' }}
        run: |
          #!/usr/bin/env bash
          set -euo pipefail
          shopt -s inherit_errexit

          make test \
            version="${{ matrix.includes.version }}" \
            tarballPath="${{ steps.make-outputdir.outputs.outputdir }}/*.tgz"

  # Add in the checksum and URI fields to the metadata if the dependency was compiled
  update-metadata:
    name: Update Metadata (if compiled)
    needs:
      - retrieve
      - get-compile-and-test
      - compile
    strategy:
      matrix:
        includes: ${{ fromJSON(needs.retrieve.outputs.compilation-json) }}
    if: ${{ needs.retrieve.outputs.compilation-length > 0 && needs.get-compile-and-test.outputs.should-compile == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Download artifact files
        uses: actions/download-artifact@v4
        with:
          name: '${{ needs.retrieve.outputs.id }}-${{ matrix.includes.version }}-${{ matrix.includes.target }}'

      - name: Get artifact file name
        id: get-file-names
        run: |
          #!/usr/bin/env bash
          set -euo pipefail
          shopt -s inherit_errexit

          echo "artifact-file=$(basename ./*.tgz)" >> "$GITHUB_OUTPUT"
          echo "checksum-file=$(basename ./*.tgz.checksum)" >> "$GITHUB_OUTPUT"

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          aws-access-key-id: ${{ secrets.AWS_S3_DEPENDENCIES_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_S3_DEPENDENCIES_SECRET_ACCESS_KEY }}
          aws-region: us-east-1

      - name: Upload to S3
        id: upload
        uses: paketo-buildpacks/github-config/actions/dependency/upload-to-s3@main
        with:
          bucket-name: "paketo-buildpacks"
          dependency-name: ${{ needs.retrieve.outputs.id }}
          artifact-path: ${{ steps.get-file-names.outputs.artifact-file }}

      - name: Get Checksum
        id: get-checksum
        run: echo "checksum=$(cat ${{ steps.get-file-names.outputs.checksum-file }})" >> "$GITHUB_OUTPUT"

      - name: Download metadata.json
        uses: actions/download-artifact@v4
        with:
          name: metadata.json

      # Create target/version specific metadata files
      # Due to limitations with the upload action, we can no longer modify/upload the same metadata file
      - name: Write dependency-specific metadata to new file
        id: dependency-metadata
        run: |
          #!/usr/bin/env bash
          set -euo pipefail
          shopt -s inherit_errexit

          metadata_file_name="${{ matrix.includes.target }}-${{ matrix.includes.version }}-metadata-file.json"
          cat metadata.json | jq -r ['.[] | select( .version == "${{ matrix.includes.version }}" and .target == "${{ matrix.includes.target }}")'] > $metadata_file_name
          echo "file=$(echo $metadata_file_name)" >> "$GITHUB_OUTPUT"

      - name: Update `checksum` and `uri` in metadata for ${{ matrix.includes.target }} ${{ matrix.includes.version }}
        if: ${{ matrix.includes.checksum == '' && matrix.includes.uri == '' }}
        uses: paketo-buildpacks/github-config/actions/dependency/update-metadata-json@main
        with:
          version: ${{ matrix.includes.version }}
          target: ${{ matrix.includes.target }}
          checksum: ${{ steps.get-checksum.outputs.checksum }}
          uri: ${{ steps.upload.outputs.dependency-uri }}
          file: ${{ steps.dependency-metadata.outputs.file }}

      - name: Upload modified metadata
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.dependency-metadata.outputs.file }}
          path: ${{ steps.dependency-metadata.outputs.file }}

  assemble:
    name: Update buildpack.toml
    needs:
      - retrieve
      - test
      - compile
      - update-metadata
    # Update buildpack.toml only if ALL of the following conditions are met:
    #   (1) Retrieval step has succeeded and has found at least 1 new version
    #   (2) Testing step has succeeded OR been skipped
    #   (3) Compilation/Testing step has succeeded OR been skipped
    #   (4) Update metadata step has succeeded OR been skipped
    if: always() && needs.retrieve.result == 'success' && needs.retrieve.outputs.length > 0 && (needs.test.result == 'success' || needs.test.result == 'skipped') && (needs.compile.result == 'success' || needs.compile.result == 'skipped') && (needs.update-metadata.result == 'success' || needs.update-metadata.result == 'skipped')
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Checkout Branch
        uses: paketo-buildpacks/github-config/actions/pull-request/checkout-branch@main
        with:
          branch: automation/dependencies/update-from-metadata

      - name: Make Temporary Artifact Directory
        id: make-outputdir
        run: echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT"


      # Metadata file for the non-compiled dependencies, if there are any
      - name: Download metadata.json file
        uses: actions/download-artifact@v4
        with:
          path: "${{ steps.make-outputdir.outputs.outputdir }}/metadata-files"
          pattern: "from-source-metadata.json"
          merge-multiple: true

      # If we compiled the dependency, and updated the metadata:
      # Download each metadata file, and combine them into one
      - name: Download individual metadata-file.json file(s)
        if: ${{ needs.update-metadata.result == 'success' }}
        uses: actions/download-artifact@v4
        with:
          path: "${{ steps.make-outputdir.outputs.outputdir }}/metadata-files"
          pattern: "*metadata-file.json"
          merge-multiple: true
      - name: Display Metadata Files
        run: ls "${{ steps.make-outputdir.outputs.outputdir }}/metadata-files"
      - name: Combine Metadata Files
        run: |
          #!/usr/bin/env bash
          set -euo pipefail
          shopt -s inherit_errexit

          jq -s 'add' ${{ steps.make-outputdir.outputs.outputdir }}/metadata-files/* > "${{ steps.make-outputdir.outputs.outputdir }}/metadata.json"

      - name: Update dependencies from metadata.json
        id: update
        uses: paketo-buildpacks/github-config/actions/dependency/update-from-metadata@main
        with:
          buildpack_toml_path: "${{ github.workspace }}/buildpack.toml"
          metadata_file_path: "${{ steps.make-outputdir.outputs.outputdir }}/metadata.json"

      - name: Show git diff
        run: |
          git diff

      - name: Commit
        id: commit
        uses: paketo-buildpacks/github-config/actions/pull-request/create-commit@main
        with:
          message: "Updating buildpack.toml with new versions ${{ steps.update.outputs.new-versions }}"
          pathspec: "."
          keyid: ${{ secrets.PAKETO_BOT_GPG_SIGNING_KEY_ID }}
          key: ${{ secrets.PAKETO_BOT_GPG_SIGNING_KEY }}

      - name: Push Branch 'automation/dependencies/update-from-metadata'
        if: ${{ steps.commit.outputs.commit_sha != '' }}
        uses: paketo-buildpacks/github-config/actions/pull-request/push-branch@main
        with:
          branch: automation/dependencies/update-from-metadata

      - name: Open Pull Request
        if: ${{ steps.commit.outputs.commit_sha != '' }}
        uses: paketo-buildpacks/github-config/actions/pull-request/open@main
        with:
          token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }}
          title: "Updates buildpack.toml with ${{ steps.update.outputs.new-versions }}"
          branch: automation/buildpack.toml/update-from-metadata

  failure:
    name: Alert on Failure
    runs-on: ubuntu-22.04
    needs: [ retrieve, get-compile-and-test, test, compile, update-metadata, assemble ]
    if: ${{ always() && needs.retrieve.result == 'failure' || needs.get-compile-and-test.result == 'failure' || needs.test.result == 'failure' || needs.compile.result == 'failure' || needs.update-metadata.result == 'failure' || needs.assemble.result == 'failure' }}
    steps:
      - name: File Failure Alert Issue
        uses: paketo-buildpacks/github-config/actions/issue/file@main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          repo: ${{ github.repository }}
          label: "failure:update-dependencies"
          comment_if_exists: true
          issue_title: "Failure: Update Dependencies workflow"
          issue_body: |
            Update Dependencies From Metadata workflow [failed](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}).
          comment_body: |
             Another failure occurred: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}