earthly.yml

name: earthly ci

on:
  pull_request:
    types: [opened, synchronize, reopened, closed]
    branches:
      - master
  workflow_dispatch:
    inputs:
      upload:
        description: "Upload the container to our registry"
        default: false
        type: boolean

env:
  AWS_REGION: "eu-central-1"
  SSH_AUTH_SOCK: /tmp/ssh_agent.sock

jobs:
  build-and-push:
    runs-on: ubuntu-latest
    if: github.event.action != 'closed' || github.event.pull_request.merged == true
    concurrency:
      group: pr-${{ github.event.pull_request.number }}-author-${{ github.event.pull_request.user.login }}
      cancel-in-progress: true
    permissions:
      id-token: write
      contents: write
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup Earthly
        uses: ./.github/earthly-setup
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
        with:
          ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }}
          config_tar: ${{ secrets.EARTHLY_TAR }}

      - name: Build With Benchmarking Features Enabled
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
        env:
          EARTHLY_CI: true
        run: |
          export EARTHLY_OUTPUT=true
          earthly -P +build --PROFILE=production --FEATURES=runtime-benchmarks

      - name: Generate and Extract Weights
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
        continue-on-error: true
        run: |
          repository_name="${GITHUB_REPOSITORY##*/}"
          echo "Listing contents on the runner host in /home/runner/work/${repository_name}/${repository_name}:"
          ls -la /home/runner/work/${repository_name}/${repository_name}
          echo "Pulling Docker image..."
          docker pull ubuntu:22.04
          mkdir -p weights
          echo "Running Docker container..."
          docker run -d --name weight_generation \
              --memory=4096m \
              --cpus=1 \
              -v /home/runner/work/${repository_name}/${repository_name}:/workspace \
              ubuntu:22.04 \
              /bin/bash -c "sleep infinity"
          echo "Installing necessary packages inside the container..."
          docker exec weight_generation bash -c "\
              apt-get update && \
              apt-get install -y jq curl build-essential && \
              echo 'Checking files in workspace...' && \
              ls -la /workspace && \
              mkdir -p /workspace/target/production && \
              cp /workspace/sidechains-substrate-node /workspace/target/production/sidechains-substrate-node && \
              echo 'Verifying the binary is in the expected path...' && \
              ls -la /workspace/target/production && \
              cd /workspace && \
              echo 'Setting the current working directory to /workspace...' && \
              chmod +x scripts/run_all_pallet_overhead_and_machine_benchmarks.sh && \
              chmod +x scripts/run_storage_benchmarks.sh && \
              source .envrc || true && \
              ./scripts/run_all_pallet_overhead_and_machine_benchmarks.sh -b && \
              ./scripts/run_storage_benchmarks.sh -b || true"
          echo "Finding and copying weight files..."
          weight_files=$(docker exec weight_generation find /workspace/runtime/src/weights -name '*.rs')
          echo "$weight_files" | while read weight_file; do
              weight_file_name=$(basename "$weight_file")
              echo "Copying ${weight_file_name}"
              docker cp "weight_generation:$weight_file" "weights/${weight_file_name}"
          done
          docker stop weight_generation
          docker rm weight_generation

      - name: Upload Weights
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
        continue-on-error: true
        uses: actions/upload-artifact@v4
        with:
          name: weights
          path: weights/

      - name: Overwrite Weights in Runtime Directory
        continue-on-error: true
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }}
        run: |
          pwd
          ls -la
          sudo chmod -R a+rwx ./runtime/src/weights
          for weight_file in weights/*.rs
          do
            cp "$weight_file" "./runtime/src/weights/$(basename "$weight_file")"
          done

      - name: Acquire AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }}
          aws-region: ${{ env.AWS_REGION }}

      - name: Login to container registry
        uses: docker/login-action@v3
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
        with:
          registry: ${{ secrets.ECR_REGISTRY_SECRET }}

      - name: Main Build
        env:
          EARTHLY_CI: true
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
        run: |
          ref=${{ github.ref_name }}
          type=${{ github.ref_type }}
          args=("--image=${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node")
          event_name="${{ github.event_name }}"
          if [[ "$type" == "tag" && "$ref" =~ ^v ]]; then
            export EARTHLY_PUSH=true
            args+=("--tags=$ref")
          elif [[ \
            "${{ github.event.pull_request.merged }}" == 'true' \
            && "$ref" == 'master' \
            || "${{ inputs.upload }}" == 'true' \
          ]]; then
            export EARTHLY_PUSH=true
            export EARTHLY_OUTPUT=true
          fi
          if [[ "$EARTHLY_PUSH" == true ]]; then
            args+=(--PROFILE=production)
          fi
          earthly -P +ci ${args[@]}

      - name: Upload chain spec artifacts
        uses: actions/upload-artifact@v4
        if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == true) }}
        with:
          name: chain-specs
          path: |
            ./devnet_chain_spec.json
            ./staging_chain_spec.json

  generate-manifest:
    needs: build-and-push
    if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Create and Push Manifest
        env:
          GH_TOKEN: ${{ secrets.ACTIONS_PAT }}
        run: |
          cd .github/workflows/argocd
          bash generate-manifest.sh ${{ github.sha }}

      - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks)
        run: sleep 720s

  run-e2e-tests:
    needs: generate-manifest
    uses: ./.github/workflows/e2e.yml
    with:
      node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local
      node-port: 9933
    secrets: inherit

  teardown:
    runs-on: ubuntu-latest
    needs: [build-and-push, generate-manifest, run-e2e-tests]
    if: always() && needs.generate-manifest.result == 'success'
    steps:
      - name: Checkout ArgoCD Repository
        uses: actions/checkout@v4
        with:
          repository: input-output-hk/sidechains-argocd
          token: ${{ secrets.ACTIONS_PAT }}
          path: sidechains-argocd

      - name: Delete Ephemeral Environment Files
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.ACTIONS_PAT }}
          script: |
            const fs = require('fs');
            const path = require('path');

            const directory = 'sidechains-argocd/integration-testing';
            const files = fs.readdirSync(directory);

            for (const file of files) {
              if (file.startsWith('manifest-sha-')) {
                console.log(`Deleting file: ${file}`);

                // Fetch the SHA of the file
                const shaResponse = await github.rest.repos.getContent({
                  owner: 'input-output-hk',
                  repo: 'sidechains-argocd',
                  path: `integration-testing/${file}`,
                });
                const sha = shaResponse.data.sha;

                // GitHub API request to delete the file
                await github.rest.repos.deleteFile({
                  owner: 'input-output-hk',
                  repo: 'sidechains-argocd',
                  path: `integration-testing/${file}`,
                  message: `ci: Tear down integration-testing environment for SHA #${file.split('-').pop().split('.')[0]}`,
                  sha: sha,
                  branch: 'main'
                });
              }
            }

  chain-specs:
    runs-on: [self-hosted, eks]
    needs: [build-and-push]
    if: github.ref_name == 'master' || inputs.upload == true
    permissions:
      id-token: write
      contents: write
    steps:
      - name: Install kubectl and awscli
        run: |
          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
          chmod +x ./kubectl
          sudo mv ./kubectl /usr/local/bin/kubectl
          sudo apt update && sudo apt install -y awscli

      - name: Configure kubectl
        run: |
          echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml
          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
          kubectl config use-context my-context

      - name: Download chain spec artifacts
        uses: actions/download-artifact@v4
        with:
          name: chain-specs
          path: ./artifacts

      - name: Update Kubernetes secret for devnet chain spec
        run: |
          TIMESTAMP=$(date +%Y%m%d%H%M)
          SHA=${{ github.sha }}
          kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc

      - name: Update Kubernetes secret for staging chain spec
        run: |
          TIMESTAMP=$(date +%Y%m%d%H%M)
          SHA=${{ github.sha }}
          kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging