diff --git a/.github/workflows/earthly.yml b/.github/workflows/earthly.yml new file mode 100644 index 000000000..59cf00807 --- /dev/null +++ b/.github/workflows/earthly.yml @@ -0,0 +1,281 @@ +name: earthly ci + +on: + pull_request: + types: [opened, synchronize, reopened, closed] + branches: + - master + workflow_dispatch: + inputs: + upload: + description: "Upload the container to our registry" + default: false + type: boolean + repository: + description: "GitHub repository name (e.g., username/repo)" + required: true + default: 'input-output-hk/partner-chains' + branch: + description: "Branch name to build" + required: true + default: 'master' +env: + AWS_REGION: "eu-central-1" + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + +jobs: + build-and-push: + runs-on: ubuntu-latest + if: github.event.action != 'closed' || github.event.pull_request.merged == true + concurrency: + group: pr-${{ github.event.pull_request.number }}-author-${{ github.event.pull_request.user.login }} + cancel-in-progress: true + permissions: + id-token: write + contents: write + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repository }} + ref: ${{ github.event.inputs.branch }} + + - name: Setup Earthly + uses: ./.github/earthly-setup + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} + with: + ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }} + config_tar: ${{ secrets.EARTHLY_TAR }} + + - name: Build With Benchmarking Features Enabled + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} + env: + EARTHLY_CI: true + run: | + export EARTHLY_OUTPUT=true + earthly -P +build --PROFILE=production --FEATURES=runtime-benchmarks + + - name: Generate and Extract Weights + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} + continue-on-error: true + run: | + repository_name="${GITHUB_REPOSITORY##*/}" + echo "Listing contents on the runner host in /home/runner/work/${repository_name}/${repository_name}:" + ls -la /home/runner/work/${repository_name}/${repository_name} + echo "Pulling Docker image..." + docker pull ubuntu:22.04 + mkdir -p weights + echo "Running Docker container..." + docker run -d --name weight_generation \ + --memory=4096m \ + --cpus=1 \ + -v /home/runner/work/${repository_name}/${repository_name}:/workspace \ + ubuntu:22.04 \ + /bin/bash -c "sleep infinity" + echo "Installing necessary packages inside the container..." + docker exec weight_generation bash -c "\ + apt-get update && \ + apt-get install -y jq curl build-essential && \ + echo 'Checking files in workspace...' && \ + ls -la /workspace && \ + mkdir -p /workspace/target/production && \ + cp /workspace/sidechains-substrate-node /workspace/target/production/sidechains-substrate-node && \ + echo 'Verifying the binary is in the expected path...' && \ + ls -la /workspace/target/production && \ + cd /workspace && \ + echo 'Setting the current working directory to /workspace...' && \ + chmod +x scripts/run_all_pallet_overhead_and_machine_benchmarks.sh && \ + chmod +x scripts/run_storage_benchmarks.sh && \ + source .envrc || true && \ + ./scripts/run_all_pallet_overhead_and_machine_benchmarks.sh -b && \ + ./scripts/run_storage_benchmarks.sh -b || true" + echo "Finding and copying weight files..." + weight_files=$(docker exec weight_generation find /workspace/runtime/src/weights -name '*.rs') + echo "$weight_files" | while read weight_file; do + weight_file_name=$(basename "$weight_file") + echo "Copying ${weight_file_name}" + docker cp "weight_generation:$weight_file" "weights/${weight_file_name}" + done + docker stop weight_generation + docker rm weight_generation + + - name: Upload Weights + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: weights + path: weights/ + + - name: Overwrite Weights in Runtime Directory + continue-on-error: true + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == 'true') }} + run: | + pwd + ls -la + sudo chmod -R a+rwx ./runtime/src/weights + for weight_file in weights/*.rs + do + cp "$weight_file" "./runtime/src/weights/$(basename "$weight_file")" + done + + - name: Acquire AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to container registry + uses: docker/login-action@v3 + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} + with: + registry: ${{ secrets.ECR_REGISTRY_SECRET }} + + - name: Main Build + env: + EARTHLY_CI: true + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') }} + run: | + ref=${{ github.ref_name }} + type=${{ github.ref_type }} + args=("--image=${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node") + event_name="${{ github.event_name }}" + if [[ "$type" == "tag" && "$ref" =~ ^v ]]; then + export EARTHLY_PUSH=true + args+=("--tags=$ref") + elif [[ \ + "${{ github.event.pull_request.merged }}" == 'true' \ + && "$ref" == 'master' \ + || "${{ inputs.upload }}" == 'true' \ + ]]; then + export EARTHLY_PUSH=true + export EARTHLY_OUTPUT=true + fi + if [[ "$EARTHLY_PUSH" == true ]]; then + args+=(--PROFILE=production) + fi + earthly -P +ci ${args[@]} + + - name: Upload chain spec artifacts + uses: actions/upload-artifact@v4 + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci-off') && (github.ref_name == 'master' || inputs.upload == true) }} + with: + name: chain-specs + path: | + ./devnet_chain_spec.json + ./staging_chain_spec.json + + generate-manifest: + needs: build-and-push + if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create and Push Manifest + env: + GH_TOKEN: ${{ secrets.ACTIONS_PAT }} + run: | + cd .github/workflows/modules/deploy/argocd + bash generate-manifest.sh ${{ github.sha }} + + - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) + run: sleep 720s + + run-e2e-tests: + needs: generate-manifest + uses: ./.github/workflows/modules/tests/argocd-tests.yml + with: + node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local + node-port: 9933 + secrets: inherit + + teardown: + runs-on: ubuntu-latest + needs: [build-and-push, generate-manifest, run-e2e-tests] + if: always() && needs.generate-manifest.result == 'success' + steps: + - name: Checkout ArgoCD Repository + uses: actions/checkout@v4 + with: + repository: input-output-hk/sidechains-argocd + token: ${{ secrets.ACTIONS_PAT }} + path: sidechains-argocd + + - name: Delete Ephemeral Environment Files + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.ACTIONS_PAT }} + script: | + const fs = require('fs'); + const path = require('path'); + + const directory = 'sidechains-argocd/integration-testing'; + const files = fs.readdirSync(directory); + + for (const file of files) { + if (file.startsWith('manifest-sha-')) { + console.log(`Deleting file: ${file}`); + + // Fetch the SHA of the file + const shaResponse = await github.rest.repos.getContent({ + owner: 'input-output-hk', + repo: 'sidechains-argocd', + path: `integration-testing/${file}`, + }); + const sha = shaResponse.data.sha; + + // GitHub API request to delete the file + await github.rest.repos.deleteFile({ + owner: 'input-output-hk', + repo: 'sidechains-argocd', + path: `integration-testing/${file}`, + message: `ci: Tear down integration-testing environment for SHA #${file.split('-').pop().split('.')[0]}`, + sha: sha, + branch: 'main' + }); + } + } + + chain-specs: + runs-on: [self-hosted, eks] + needs: [build-and-push] + if: github.ref_name == 'master' || inputs.upload == true + permissions: + id-token: write + contents: write + steps: + - name: Install kubectl and awscli + run: | + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + sudo apt update && sudo apt install -y awscli + + - name: Configure kubectl + run: | + echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml + kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true + kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} + kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default + kubectl config use-context my-context + + - name: Download chain spec artifacts + uses: actions/download-artifact@v4 + with: + name: chain-specs + path: ./artifacts + + - name: Update Kubernetes secret for devnet chain spec + run: | + TIMESTAMP=$(date +%Y%m%d%H%M) + SHA=${{ github.sha }} + kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc + + - name: Update Kubernetes secret for staging chain spec + run: | + TIMESTAMP=$(date +%Y%m%d%H%M) + SHA=${{ github.sha }} + kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging \ No newline at end of file diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 000000000..a887bf9b4 --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,46 @@ +name: "Nix Devshells CI" +on: + pull_request: + push: + branches: + - master + workflow_dispatch: + repository_dispatch: + types: [test-run-pr] + +env: + AWS_DEFAULT_REGION: eu-central-1 + +jobs: + build-and-test: + permissions: + id-token: write + contents: read + strategy: + matrix: + os: [nixos, macos] + runs-on: + - self-hosted + - ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ github.token }} + - name: Acquire AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Add signing key for nix + run: echo "${{ secrets.NIX_SIGNING_KEY }}" > "${{ runner.temp }}/nix-key" + - name: Run nixci to build/test all outputs + run: | + nix run github:srid/nixci -- -v build -- --fallback > /tmp/outputs + - name: Copy nix scopes to nix cache + run: | + nix-store --stdin -q --deriver < /tmp/outputs | nix-store --stdin -qR --include-outputs \ + | nix copy --stdin --to \ + "s3://cache.sc.iog.io?secret-key=${{ runner.temp }}/nix-key®ion=$AWS_DEFAULT_REGION" \ + && rm /tmp/outputs \ No newline at end of file diff --git a/.github/workflows/publish-doc.yml b/.github/workflows/publish-doc.yml new file mode 100644 index 000000000..2ed0026a4 --- /dev/null +++ b/.github/workflows/publish-doc.yml @@ -0,0 +1,63 @@ +name: Publish Documentation + +on: + + push: + branches: + - master + pull_request: + branches: + - master + workflow_dispatch: + inputs: + rustdoc: + description: "Build and publish the rustdoc" + default: false + type: boolean + +env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + +jobs: + deploy-rustdoc: + if: github.event_name != 'workflow_dispatch' || inputs.rustdoc + name: Deploy Rust Docs + runs-on: ubuntu-latest + + steps: + - name: Install tooling + run: | + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Checkout repository + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + + - name: Add SSH key to read Substrate Repo + run: | + mkdir ~/.ssh + ssh-keyscan github.com >> ~/.ssh/known_hosts + ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null + ssh-add - <<< "${{ secrets.SUBSTRATE_REPO_SSH_KEY }}" + + # With rustup's nice new toml format, we just need to run rustup show to install the toolchain + # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 + - name: Rust versions + run: rustup show + + - name: Rust cache + uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # v2.6.2 + + - name: Build rustdocs + run: SKIP_WASM_BUILD=1 cargo doc --all --no-deps + + - name: Make index.html + run: echo "" > ./target/doc/index.html + + - name: Deploy documentation + if: github.ref_name == 'master' + uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: gh-pages + publish_dir: ./target/doc \ No newline at end of file