fix: install cargo-chef unconditionally - ETCM-7736 #309
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: earthly ci | |
| on: | |
| pull_request: | |
| types: [opened, synchronize, reopened, closed] | |
| branches: | |
| - master | |
| workflow_dispatch: | |
| inputs: | |
| upload: | |
| description: "Upload the container to our registry" | |
| default: false | |
| type: boolean | |
| env: | |
| AWS_REGION: "eu-central-1" | |
| SSH_AUTH_SOCK: /tmp/ssh_agent.sock | |
| jobs: | |
| build-and-push: | |
| runs-on: ubuntu-latest | |
| if: github.event.action != 'closed' || github.event.pull_request.merged == true | |
| concurrency: | |
| group: pr-${{ github.event.pull_request.number }}-author-${{ github.event.pull_request.user.login }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Earthly | |
| uses: ./.github/earthly-setup | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| with: | |
| ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }} | |
| config_tar: ${{ secrets.EARTHLY_TAR }} | |
| - name: Build With Benchmarking Features Enabled | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| env: | |
| EARTHLY_CI: true | |
| run: | | |
| export EARTHLY_OUTPUT=true | |
| earthly -P +build --PROFILE=production --FEATURES=runtime-benchmarks | |
| - name: Generate and Extract Weights | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| continue-on-error: true | |
| run: | | |
| repository_name="${GITHUB_REPOSITORY##*/}" | |
| echo "Listing contents on the runner host in /home/runner/work/${repository_name}/${repository_name}:" | |
| ls -la /home/runner/work/${repository_name}/${repository_name} | |
| echo "Pulling Docker image..." | |
| docker pull ubuntu:22.04 | |
| mkdir -p weights | |
| echo "Running Docker container..." | |
| docker run -d --name weight_generation \ | |
| --memory=4096m \ | |
| --cpus=1 \ | |
| -v /home/runner/work/${repository_name}/${repository_name}:/workspace \ | |
| ubuntu:22.04 \ | |
| /bin/bash -c "sleep infinity" | |
| echo "Installing necessary packages inside the container..." | |
| docker exec weight_generation bash -c "\ | |
| apt-get update && \ | |
| apt-get install -y jq curl build-essential && \ | |
| echo 'Checking files in workspace...' && \ | |
| ls -la /workspace && \ | |
| mkdir -p /workspace/target/production && \ | |
| cp /workspace/sidechains-substrate-node /workspace/target/production/sidechains-substrate-node && \ | |
| echo 'Verifying the binary is in the expected path...' && \ | |
| ls -la /workspace/target/production && \ | |
| cd /workspace && \ | |
| echo 'Setting the current working directory to /workspace...' && \ | |
| chmod +x scripts/run_all_pallet_overhead_and_machine_benchmarks.sh && \ | |
| chmod +x scripts/run_storage_benchmarks.sh && \ | |
| source .envrc || true && \ | |
| ./scripts/run_all_pallet_overhead_and_machine_benchmarks.sh -b && \ | |
| ./scripts/run_storage_benchmarks.sh -b || true" | |
| echo "Finding and copying weight files..." | |
| weight_files=$(docker exec weight_generation find /workspace/runtime/src/weights -name '*.rs') | |
| echo "$weight_files" | while read weight_file; do | |
| weight_file_name=$(basename "$weight_file") | |
| echo "Copying ${weight_file_name}" | |
| docker cp "weight_generation:$weight_file" "weights/${weight_file_name}" | |
| done | |
| docker stop weight_generation | |
| docker rm weight_generation | |
| - name: Upload Weights | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| continue-on-error: true | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: weights | |
| path: weights/ | |
| - name: Overwrite Weights in Runtime Directory | |
| continue-on-error: true | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} | |
| run: | | |
| pwd | |
| ls -la | |
| sudo chmod -R a+rwx ./runtime/src/weights | |
| for weight_file in weights/*.rs | |
| do | |
| cp "$weight_file" "./runtime/src/weights/$(basename "$weight_file")" | |
| done | |
| - name: Acquire AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| with: | |
| role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Login to container registry | |
| uses: docker/login-action@v3 | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| with: | |
| registry: ${{ secrets.ECR_REGISTRY_SECRET }} | |
| - name: Main Build | |
| env: | |
| EARTHLY_CI: true | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| run: | | |
| ref=${{ github.ref_name }} | |
| type=${{ github.ref_type }} | |
| args=("--image=${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node") | |
| event_name="${{ github.event_name }}" | |
| if [[ "$type" == "tag" && "$ref" =~ ^v ]]; then | |
| export EARTHLY_PUSH=true | |
| args+=("--tags=$ref") | |
| elif [[ \ | |
| "${{ github.event.pull_request.merged }}" == 'true' \ | |
| && "$ref" == 'master' \ | |
| || "${{ inputs.upload }}" == 'true' \ | |
| ]]; then | |
| export EARTHLY_PUSH=true | |
| export EARTHLY_OUTPUT=true | |
| fi | |
| if [[ "$EARTHLY_PUSH" == true ]]; then | |
| args+=(--PROFILE=production) | |
| fi | |
| earthly -P +ci ${args[@]} | |
| - name: Upload chain spec artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == true) }} | |
| with: | |
| name: chain-specs | |
| path: | | |
| ./devnet_chain_spec.json | |
| ./staging_chain_spec.json | |
| generate-manifest: | |
| needs: build-and-push | |
| if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Create and Push Manifest | |
| env: | |
| GH_TOKEN: ${{ secrets.ACTIONS_PAT }} | |
| run: | | |
| cd .github/workflows/argocd | |
| bash generate-manifest.sh ${{ github.sha }} | |
| - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) | |
| run: sleep 720s | |
| run-e2e-tests: | |
| needs: generate-manifest | |
| uses: ./.github/workflows/e2e.yml | |
| with: | |
| node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local | |
| node-port: 9933 | |
| secrets: inherit | |
| teardown: | |
| runs-on: ubuntu-latest | |
| needs: [build-and-push, generate-manifest, run-e2e-tests] | |
| if: always() && needs.generate-manifest.result == 'success' | |
| steps: | |
| - name: Checkout ArgoCD Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: input-output-hk/sidechains-argocd | |
| token: ${{ secrets.ACTIONS_PAT }} | |
| path: sidechains-argocd | |
| - name: Delete Ephemeral Environment Files | |
| uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{ secrets.ACTIONS_PAT }} | |
| script: | | |
| const fs = require('fs'); | |
| const path = require('path'); | |
| const directory = 'sidechains-argocd/integration-testing'; | |
| const files = fs.readdirSync(directory); | |
| for (const file of files) { | |
| if (file.startsWith('manifest-sha-')) { | |
| console.log(`Deleting file: ${file}`); | |
| // Fetch the SHA of the file | |
| const shaResponse = await github.rest.repos.getContent({ | |
| owner: 'input-output-hk', | |
| repo: 'sidechains-argocd', | |
| path: `integration-testing/${file}`, | |
| }); | |
| const sha = shaResponse.data.sha; | |
| // GitHub API request to delete the file | |
| await github.rest.repos.deleteFile({ | |
| owner: 'input-output-hk', | |
| repo: 'sidechains-argocd', | |
| path: `integration-testing/${file}`, | |
| message: `ci: Tear down integration-testing environment for SHA #${file.split('-').pop().split('.')[0]}`, | |
| sha: sha, | |
| branch: 'main' | |
| }); | |
| } | |
| } | |
| chain-specs: | |
| runs-on: [self-hosted, eks] | |
| needs: [build-and-push] | |
| if: github.ref_name == 'master' || inputs.upload == true | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Install kubectl and awscli | |
| run: | | |
| curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" | |
| chmod +x ./kubectl | |
| sudo mv ./kubectl /usr/local/bin/kubectl | |
| sudo apt update && sudo apt install -y awscli | |
| - name: Configure kubectl | |
| run: | | |
| echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml | |
| kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true | |
| kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} | |
| kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default | |
| kubectl config use-context my-context | |
| - name: Download chain spec artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: chain-specs | |
| path: ./artifacts | |
| - name: Update Kubernetes secret for devnet chain spec | |
| run: | | |
| TIMESTAMP=$(date +%Y%m%d%H%M) | |
| SHA=${{ github.sha }} | |
| kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc | |
| - name: Update Kubernetes secret for staging chain spec | |
| run: | | |
| TIMESTAMP=$(date +%Y%m%d%H%M) | |
| SHA=${{ github.sha }} | |
| kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging |