From 3b4c996f5fc5c29caabc5b734732e5ab0613c055 Mon Sep 17 00:00:00 2001 From: Skylar Simoncelli Date: Thu, 19 Sep 2024 17:21:35 +0300 Subject: [PATCH] feat: more modules --- .github/workflows/cd.yml | 168 ++++---------- .github/workflows/ci.yml | 216 +++--------------- .../modules/artifacts/build-pcsc-artifact.yml | 2 + .../artifacts/download-pcsc-artifact.yml | 58 +++-- .../artifacts/generate-chain-specs.yml | 46 ++++ .../modules/deploy/argocd/deploy-argocd.yml | 29 +++ .../{ => deploy}/argocd/generate-manifest.sh | 0 .../modules/{ => deploy}/argocd/manifest.yaml | 0 .../modules/deploy/argocd/teardown-argocd.yml | 56 +++++ .../modules/deploy/deploy-rustdoc.yml | 54 +++++ .../upload-chain-specs.yml} | 45 ++-- .../modules/release/create-draft-release.yml | 111 +++++++++ .../modules/release/publish-release.yml | 40 ++++ .../tests/{e2e.yml => argocd-tests.yml} | 0 .../modules/tests/devshell-tests.tml | 55 +++++ .../modules/tests/staging-preprod-tests.yml | 0 .../modules/tests/staging-preview-tests.yml | 0 17 files changed, 534 insertions(+), 346 deletions(-) create mode 100644 .github/workflows/modules/artifacts/generate-chain-specs.yml create mode 100644 .github/workflows/modules/deploy/argocd/deploy-argocd.yml rename .github/workflows/modules/{ => deploy}/argocd/generate-manifest.sh (100%) rename .github/workflows/modules/{ => deploy}/argocd/manifest.yaml (100%) create mode 100644 .github/workflows/modules/deploy/argocd/teardown-argocd.yml create mode 100644 .github/workflows/modules/deploy/deploy-rustdoc.yml rename .github/workflows/modules/{chain-specs.yml => deploy/upload-chain-specs.yml} (65%) create mode 100644 .github/workflows/modules/release/create-draft-release.yml create mode 100644 .github/workflows/modules/release/publish-release.yml rename .github/workflows/modules/tests/{e2e.yml => argocd-tests.yml} (100%) create mode 100644 .github/workflows/modules/tests/devshell-tests.tml create mode 100644 .github/workflows/modules/tests/staging-preprod-tests.yml create mode 100644 .github/workflows/modules/tests/staging-preview-tests.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 4cf2de022..72b05d643 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,8 +1,4 @@ -name: Build, test, release and deploy - -permissions: - id-token: write - contents: write +name: CD on: workflow_dispatch: @@ -16,6 +12,10 @@ on: required: true type: string +permissions: + id-token: write + contents: write + env: AWS_REGION: "eu-central-1" SSH_AUTH_SOCK: /tmp/ssh_agent.sock @@ -39,26 +39,9 @@ jobs: tag: ${{ inputs.tag }} partner-chains-smart-contracts: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - name: Extract PCSC Release Info from flake.nix - id: extract-release - run: | - echo "Extracting PCSC release version from flake.nix..." - release=$(cat flake.nix | grep -Po 'url = "github:input-output-hk/partner-chains-smart-contracts/v\K[0-9.]+(?=";)') - echo "Release version: $release" - echo "::set-output name=release::$release" - - - name: Call Download Artifact Workflow - uses: ./.github/workflows/modules/artifacts/download-pcsc-artifact.yml - with: - release: ${{ steps.extract-release.outputs.release }} - artifact: "Specify-your-artifact-name-here" + uses: ./.github/workflows/modules/artifacts/download-pcsc-artifact.yml + with: + sha: ${{ inputs.sha }} local-environment-tests: needs: [build-pc-artifacts, partner-chains-smart-contracts] @@ -67,121 +50,52 @@ jobs: tag: ${{ inputs.tag }} image: ${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node:${{ inputs.sha }} - pre-release-candidate: - runs-on: ubuntu-latest - needs: [local-environment] - steps: - - name: Set filename variables - id: set-filenames - run: | - echo "PARTNER_CHAINS_CLI_X86_64_LINUX=partner-chains-cli-${{ inputs.tag }}-x86_64-linux" >> $GITHUB_ENV - echo "PARTNER_CHAINS_NODE_X86_64_LINUX=partner-chains-node-${{ inputs.tag }}-x86_64-linux" >> $GITHUB_ENV - echo "PARTNER_CHAINS_CLI_X86_64_APPLE_DARWIN=partner-chains-cli-${{ inputs.tag }}-x86_64-apple-darwin" >> $GITHUB_ENV - echo "PARTNER_CHAINS_NODE_X86_64_APPLE_DARWIN=partner-chains-node-${{ inputs.tag }}-x86_64-apple-darwin" >> $GITHUB_ENV - echo "PARTNER_CHAINS_CLI_AARCH64_APPLE_DARWIN=partner-chains-cli-${{ inputs.tag }}-aarch64-apple-darwin" >> $GITHUB_ENV - echo "PARTNER_CHAINS_NODE_AARCH64_APPLE_DARWIN=partner-chains-node-${{ inputs.tag }}-aarch64-apple-darwin" >> $GITHUB_ENV - - - name: Download Linux CLI artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.PARTNER_CHAINS_CLI_X86_64_LINUX }} - path: artifact-linux/ - - - name: Download Linux NODE artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.PARTNER_CHAINS_NODE_X86_64_LINUX }} - path: artifact-linux/ - - - name: Download macOS x86_64 CLI artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.PARTNER_CHAINS_CLI_X86_64_APPLE_DARWIN }} - path: artifact-macos-x86_64/ - - - name: Download macOS x86_64 NODE artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.PARTNER_CHAINS_NODE_X86_64_APPLE_DARWIN }} - path: artifact-macos-x86_64/ - - - name: Download macOS ARM64 CLI artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.PARTNER_CHAINS_CLI_AARCH64_APPLE_DARWIN }} - path: artifact-macos-arm64/ - - - name: Download macOS ARM64 NODE artifact - uses: actions/download-artifact@v4 - with: - name: ${{ env.PARTNER_CHAINS_NODE_AARCH64_APPLE_DARWIN }} - path: artifact-macos-arm64/ - - - name: Check if release already exists - id: check_release - run: | - tag="${{ inputs.tag }}" - release_response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/${{ github.repository }}/releases/tags/$tag") - if echo "$release_response" | grep -q '"message": "Not Found"'; then - echo "release_exists=false" >> $GITHUB_ENV - echo "::set-output name=release_exists::false" - else - echo "release_exists=true" >> $GITHUB_ENV - echo "::set-output name=release_exists::true" - echo "release_id=$(echo $release_response | jq -r .id)" >> $GITHUB_ENV - echo "::set-output name=release_id::$(echo $release_response | jq -r .id)" - fi - - - name: Create draft release - id: create_release - if: ${{ steps.check_release.outputs.release_exists == 'false' }} - run: | - tag="${{ inputs.tag }}" - release_response=$(curl -s -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -d '{"tag_name": "'$tag'", "name": "'$tag'", "body": "Draft release for '$tag'", "draft": true}' \ - "https://api.github.com/repos/${{ github.repository }}/releases") - echo "release_id=$(echo $release_response | jq -r .id)" >> $GITHUB_ENV - echo "::set-output name=release_id::$(echo $release_response | jq -r .id)" - - - name: Upload artifacts to release - if: ${{ steps.check_release.outputs.release_exists == 'true' || steps.create_release.outputs.release_id != '' }} - run: | - release_id="${{ steps.create_release.outputs.release_id }}" - if [ -z "$release_id" ]; then - release_id="${{ steps.check_release.outputs.release_id }}" - fi - - for artifact in "artifact-linux/${{ env.PARTNER_CHAINS_CLI_X86_64_LINUX }}" \ - "artifact-linux/${{ env.PARTNER_CHAINS_NODE_X86_64_LINUX }}" \ - "artifact-macos-x86_64/${{ env.PARTNER_CHAINS_CLI_X86_64_APPLE_DARWIN }}" \ - "artifact-macos-x86_64/${{ env.PARTNER_CHAINS_NODE_X86_64_APPLE_DARWIN }}" \ - "artifact-macos-arm64/${{ env.PARTNER_CHAINS_CLI_AARCH64_APPLE_DARWIN }}" \ - "artifact-macos-arm64/${{ env.PARTNER_CHAINS_NODE_AARCH64_APPLE_DARWIN }}"; do - chmod +x "$artifact" - curl -s -X POST \ - -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Content-Type: application/octet-stream" \ - --data-binary @"$artifact" \ - "https://uploads.github.com/repos/${{ github.repository }}/releases/$release_id/assets?name=$(basename $artifact)" - done + create-draft-release: + uses: ./.github/workflows/modules/release/create-draft-release.yml + with: + tag: ${{ inputs.tag }} + secrets: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} chain-specs: needs: build-pc-artifacts - uses: ./.github/workflows/modules/chain-specs.yml + uses: ./.github/workflows/modules/artifacts/generate-chain-specs.yml with: sha: ${{ inputs.sha }} - tag: ${{ inputs.tag }} deploy-staging-preview: needs: chain-specs uses: ./.github/workflows/modules/deploy/staging-preview-deploy.yml with: image: ${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node:${{ inputs.sha }} - chain-spec-secret: staging-chain-spec-${{ inputs.sha }} + chain-spec-secret: staging-preview-chain-spec-${{ inputs.sha }} + + staging-preview-tests: + needs: deploy-staging-preview + uses: ./.github/workflows/modules/tests/staging-preview-tests.yml build-and-publish-ghcr-image: + needs: staging-preview-tests uses: ./.github/workflows/modules/images/build-and-publish-ghcr-image.yml with: sha: ${{ inputs.sha }} - tag: ${{ inputs.tag }} \ No newline at end of file + tag: ${{ inputs.tag }} + + publish-release: + needs: staging-preview-tests + uses: ./.github/workflows/modules/release/publish-release.yml + with: + tag: ${{ inputs.tag }} + secrets: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + deploy-staging-preprod: + needs: staging-preview-tests + uses: ./.github/workflows/modules/deploy/staging-preprod-deploy.yml + with: + image: ${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node:${{ inputs.sha }} + chain-spec-secret: staging-preprod-chain-spec-${{ inputs.sha }} + + staging-preprod-tests: + needs: deploy-staging-preprod + uses: ./.github/workflows/modules/tests/staging-preprod-tests.yml \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74d2144b9..05071936e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Main Workflow +name: CI on: push: @@ -177,198 +177,52 @@ jobs: ./devnet_chain_spec.json ./staging_chain_spec.json - generate-manifest: + deploy-argocd: needs: build-and-push if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create and Push Manifest - env: - GH_TOKEN: ${{ secrets.ACTIONS_PAT }} - run: | - cd .github/workflows/modules/argocd - bash generate-manifest.sh ${{ github.sha }} - - - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) - run: sleep 720s + uses: ./.github/workflows/modules/deploy/argocd/deploy-argocd.yml + with: + sha: ${{ github.sha }} - run-e2e-tests: - needs: generate-manifest - uses: ./.github/workflows/modules/tests/e2e.yml + argocd-tests: + needs: deploy-argocd + uses: ./.github/workflows/modules/tests/argocd-tests.yml with: node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local node-port: 9933 secrets: inherit - teardown: - runs-on: ubuntu-latest - needs: [build-and-push, generate-manifest, run-e2e-tests] + teardown-argocd: + needs: [build-and-push, deploy-argocd, argocd-tests] if: ${{ always() && needs.generate-manifest.result == 'success' }} - steps: - - name: Checkout ArgoCD Repository - uses: actions/checkout@v4 - with: - repository: input-output-hk/sidechains-argocd - token: ${{ secrets.ACTIONS_PAT }} - path: sidechains-argocd - - - name: Delete Ephemeral Environment Files - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.ACTIONS_PAT }} - script: | - const fs = require('fs'); - const path = require('path'); - - const directory = 'sidechains-argocd/integration-testing'; - const files = fs.readdirSync(directory); - - for (const file of files) { - if (file.startsWith('manifest-sha-')) { - console.log(`Deleting file: ${file}`); - - // Fetch the SHA of the file - const shaResponse = await github.rest.repos.getContent({ - owner: 'input-output-hk', - repo: 'sidechains-argocd', - path: `integration-testing/${file}`, - }); - const sha = shaResponse.data.sha; + uses: ./.github/workflows/modules/deploy/teardown-argocd.yml + with: + sha: ${{ github.sha }} - // GitHub API request to delete the file - await github.rest.repos.deleteFile({ - owner: 'input-output-hk', - repo: 'sidechains-argocd', - path: `integration-testing/${file}`, - message: `ci: Tear down integration-testing environment for SHA #${file.split('-').pop().split('.')[0]}`, - sha: sha, - branch: 'main' - }); - } - } - - chain-specs: - runs-on: [self-hosted, eks] + upload-chain-specs: needs: [build-and-push] if: ${{ github.ref_name == 'master' || inputs.upload == 'true' }} - permissions: - id-token: write - contents: write - steps: - - name: Install kubectl and awscli - run: | - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - sudo apt update && sudo apt install -y awscli - - - name: Configure kubectl - run: | - echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml - kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true - kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} - kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default - kubectl config use-context my-context - - - name: Download chain spec artifacts - uses: actions/download-artifact@v4 - with: - name: chain-specs - path: ./artifacts - - - name: Update Kubernetes secret for devnet chain spec - run: | - TIMESTAMP=$(date +%Y%m%d%H%M) - SHA=${{ github.sha }} - kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc - - - name: Update Kubernetes secret for staging chain spec - run: | - TIMESTAMP=$(date +%Y%m%d%H%M) - SHA=${{ github.sha }} - kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging + uses: ./.github/workflows/modules/deploy/upload-chain-specs.yml + with: + sha: ${{ github.sha }} deploy-rustdoc: if: ${{ github.event_name != 'workflow_dispatch' || inputs.rustdoc == 'true' }} - runs-on: ubuntu-latest - steps: - - name: Install tooling - run: | - sudo apt-get install -y protobuf-compiler - protoc --version - - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Add SSH key to read Substrate Repo - run: | - mkdir ~/.ssh - ssh-keyscan github.com >> ~/.ssh/known_hosts - ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null - ssh-add - <<< "${{ secrets.SUBSTRATE_REPO_SSH_KEY }}" - - - name: Rust versions - run: rustup show - - - name: Rust cache - uses: Swatinem/rust-cache@v2.6.2 - - - name: Build rustdocs - run: SKIP_WASM_BUILD=1 cargo doc --all --no-deps - - - name: Make index.html - run: echo "" > ./target/doc/index.html - - - name: Deploy documentation - if: ${{ github.ref_name == 'master' }} - uses: peaceiris/actions-gh-pages@v3.9.3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_branch: gh-pages - publish_dir: ./target/doc - - build-and-test: - permissions: - id-token: write - contents: read - strategy: - matrix: - os: [nixos, macos] - runs-on: - - self-hosted - - ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ github.token }} - - - name: Update flake.lock with calling PR - if: ${{ github.event_name == 'repository_dispatch' }} - run: | - nix flake lock --update-input trustless-sidechain \ - --override-input trustless-sidechain \ - github:input-output-hk/partner-chains-smart-contracts/${{ github.event.client_payload.ref }} - - - name: Acquire AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - - name: Add signing key for nix - run: echo "${{ secrets.NIX_SIGNING_KEY }}" > "${{ runner.temp }}/nix-key" - - - name: Run nixci to build/test all outputs - run: | - nix run github:srid/nixci -- -v build -- --fallback > /tmp/outputs - - - name: Copy nix scopes to nix cache - run: | - nix-store --stdin -q --deriver < /tmp/outputs | nix-store --stdin -qR --include-outputs \ - | nix copy --stdin --to \ - "s3://cache.sc.iog.io?secret-key=${{ runner.temp }}/nix-key®ion=$AWS_DEFAULT_REGION" \ - && rm /tmp/outputs + uses: ./.github/workflows/modules/deploy/deploy-rustdoc.yml + with: + ssh_key: ${{ secrets.SUBSTRATE_REPO_SSH_KEY }} + secrets: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + devshell-tests: + permissions: + id-token: write + contents: read + uses: ./.github/workflows/modules/tests/devshell-tests.yml + secrets: + AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} + NIX_SIGNING_KEY: ${{ secrets.NIX_SIGNING_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }} + diff --git a/.github/workflows/modules/artifacts/build-pcsc-artifact.yml b/.github/workflows/modules/artifacts/build-pcsc-artifact.yml index e9eb6045b..2c39df8d5 100644 --- a/.github/workflows/modules/artifacts/build-pcsc-artifact.yml +++ b/.github/workflows/modules/artifacts/build-pcsc-artifact.yml @@ -11,6 +11,8 @@ on: description: "partner-chains-smart-contracts commit SHA or branch to build from" jobs: + + partner-chains-smart-contracts-x86_64-linux: runs-on: [self-hosted, nixos] steps: diff --git a/.github/workflows/modules/artifacts/download-pcsc-artifact.yml b/.github/workflows/modules/artifacts/download-pcsc-artifact.yml index 2f588397f..097f822a5 100644 --- a/.github/workflows/modules/artifacts/download-pcsc-artifact.yml +++ b/.github/workflows/modules/artifacts/download-pcsc-artifact.yml @@ -1,20 +1,44 @@ -name: Download PCSC Artifact +jobs: + parse-flake-and-download-release: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} -on: - workflow_call: - inputs: - release: - description: "release name" - artifact: - description: "artifact name" + - name: Extract PCSC Release Info from flake.nix + id: extract-release + run: | + echo "Extracting PCSC release version from flake.nix..." + release=$(grep -Po 'url = "github:input-output-hk/partner-chains-smart-contracts/v\K[0-9.]+(?=";)' flake.nix) + echo "Release version: v$release" + echo "::set-output name=release::v$release" -jobs: - - name: Download Artifact from GitHub Release - run: | - wget "https://github.com/input-output-hk/partner-chains-smart-contracts/releases/download/${{ inputs.release }}/${{ inputs.artifact }}" + - name: Construct Artifact Name + id: construct-artifact + run: | + version_without_v=${{ steps.extract-release.outputs.release#v }} + artifact="trustless-sidechain-cli-${version_without_v}-x86_64-linux.zip" + echo "Constructed artifact name: $artifact" + echo "::set-output name=artifact::$artifact" + + - name: Download Artifact as zipped.zip + run: | + wget -O zipped.zip "https://github.com/input-output-hk/partner-chains-smart-contracts/releases/download/${{ steps.extract-release.outputs.release }}/${{ steps.construct-artifact.outputs.artifact }}" + + - name: Extract zipped.zip to a temporary directory + run: | + mkdir temp_dir + unzip zipped.zip -d temp_dir + + - name: Rename extracted directory to partner-chains-smart-contracts + run: | + original_dir=$(ls temp_dir) + mv "temp_dir/$original_dir" partner-chains-smart-contracts - - name: Upload Artifact - uses: actions/upload-artifact@v4 - with: - name: partner-chains-smart-contracts-artifact - path: ${{ inputs.artifact }} \ No newline at end of file + - name: Upload Extracted Artifact + uses: actions/upload-artifact@v4 + with: + name: partner-chains-smart-contracts-artifact + path: partner-chains-smart-contracts diff --git a/.github/workflows/modules/artifacts/generate-chain-specs.yml b/.github/workflows/modules/artifacts/generate-chain-specs.yml new file mode 100644 index 000000000..d4cce74f4 --- /dev/null +++ b/.github/workflows/modules/artifacts/generate-chain-specs.yml @@ -0,0 +1,46 @@ +name: Generate Chain Specs from Node Binary + +on: + workflow_call: + inputs: + sha: + description: 'Commit SHA to append to chain spec secret name' + required: true + type: string + +jobs: + chain-specs: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + steps: + - name: Download Linux partner-chains-node artifact + uses: actions/download-artifact@v4 + with: + name: partner-chains-node-x86_64-linux-artifact-artifact + path: ./ + + - name: Generate Chain Specs + run: | + chmod +x ./partner-chains-node + source ./devnet/.envrc + ./partner-chains-node build-spec --chain local --disable-default-bootnode --raw > devnet_chain_spec.json + source ./staging/.envrc + ./partner-chains-node build-spec --chain staging --disable-default-bootnode --raw > staging_chain_spec.json + source ./staging-preview/.envrc + ./partner-chains-node build-spec --chain staging-preview --disable-default-bootnode --raw > staging_preview_chain_spec.json + source ./staging-preprod/.envrc + ./partner-chains-node build-spec --chain staging-preprod --disable-default-bootnode --raw > staging_preprod_chain_spec.json + + - name: Upload Chain Specs for both chains + uses: actions/upload-artifact@v4 + with: + name: chain-specs + path: | + devnet_chain_spec.json + staging_chain_spec.json + staging_preview_chain_spec.json + staging_preprod_chain_spec.json + + diff --git a/.github/workflows/modules/deploy/argocd/deploy-argocd.yml b/.github/workflows/modules/deploy/argocd/deploy-argocd.yml new file mode 100644 index 000000000..38538e615 --- /dev/null +++ b/.github/workflows/modules/deploy/argocd/deploy-argocd.yml @@ -0,0 +1,29 @@ +name: Deploy ArgoCD Node + +on: + workflow_call: + inputs: + sha: + description: 'Commit SHA' + required: true + type: string + +jobs: + deploy-argocd: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create and Push Manifest + env: + GH_TOKEN: ${{ secrets.ACTIONS_PAT }} + run: | + cd .github/workflows/modules/deploy/argocd + bash generate-manifest.sh ${{ inputs.sha }} + + - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) + run: sleep 720s \ No newline at end of file diff --git a/.github/workflows/modules/argocd/generate-manifest.sh b/.github/workflows/modules/deploy/argocd/generate-manifest.sh similarity index 100% rename from .github/workflows/modules/argocd/generate-manifest.sh rename to .github/workflows/modules/deploy/argocd/generate-manifest.sh diff --git a/.github/workflows/modules/argocd/manifest.yaml b/.github/workflows/modules/deploy/argocd/manifest.yaml similarity index 100% rename from .github/workflows/modules/argocd/manifest.yaml rename to .github/workflows/modules/deploy/argocd/manifest.yaml diff --git a/.github/workflows/modules/deploy/argocd/teardown-argocd.yml b/.github/workflows/modules/deploy/argocd/teardown-argocd.yml new file mode 100644 index 000000000..4decf3aff --- /dev/null +++ b/.github/workflows/modules/deploy/argocd/teardown-argocd.yml @@ -0,0 +1,56 @@ +name: Teardown ArgoCD Environment + +on: + workflow_call: + inputs: + sha: + description: "SHA of the commit" + required: true + type: string + +jobs: + teardown: + runs-on: ubuntu-latest + steps: + - name: Checkout ArgoCD Repository + uses: actions/checkout@v4 + with: + repository: input-output-hk/sidechains-argocd + token: ${{ secrets.ACTIONS_PAT }} + path: sidechains-argocd + + - name: Delete Ephemeral Environment File + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.ACTIONS_PAT }} + script: | + const fs = require('fs'); + const path = require('path'); + + const directory = 'sidechains-argocd/integration-testing'; + const targetFile = `manifest-sha-${{ inputs.sha }}.yaml`; + const filePath = path.join(directory, targetFile); + + if (fs.existsSync(filePath)) { + console.log(`Deleting file: ${targetFile}`); + + // Fetch the SHA of the file + const shaResponse = await github.rest.repos.getContent({ + owner: 'input-output-hk', + repo: 'sidechains-argocd', + path: `integration-testing/${targetFile}`, + }); + const fileSha = shaResponse.data.sha; + + // GitHub API request to delete the file + await github.rest.repos.deleteFile({ + owner: 'input-output-hk', + repo: 'sidechains-argocd', + path: `integration-testing/${targetFile}`, + message: `ci: Tear down integration-testing environment for SHA #${{ inputs.sha }}`, + sha: fileSha, + branch: 'main' + }); + } else { + console.log(`File not found: ${targetFile}`); + } diff --git a/.github/workflows/modules/deploy/deploy-rustdoc.yml b/.github/workflows/modules/deploy/deploy-rustdoc.yml new file mode 100644 index 000000000..b5d9f3679 --- /dev/null +++ b/.github/workflows/modules/deploy/deploy-rustdoc.yml @@ -0,0 +1,54 @@ +name: Deploy Rust Docs + +on: + workflow_call: + inputs: + ssh_key: + description: 'SSH key to read Substrate Repo' + required: true + type: secret + secrets: + GITHUB_TOKEN: + required: true + +jobs: + deploy-rustdoc: + runs-on: ubuntu-latest + env: + SSH_AUTH_SOCK: /tmp/ssh_agent.sock + steps: + - name: Install tooling + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Add SSH key to read Substrate Repo + run: | + mkdir -p ~/.ssh + ssh-keyscan github.com >> ~/.ssh/known_hosts + ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null + ssh-add - <<< "${{ inputs.ssh_key }}" + + - name: Rust versions + run: rustup show + + - name: Rust cache + uses: Swatinem/rust-cache@v2.6.2 + + - name: Build rustdocs + run: SKIP_WASM_BUILD=1 cargo doc --all --no-deps + + - name: Make index.html + run: echo "" > ./target/doc/index.html + + - name: Deploy documentation + if: ${{ github.ref_name == 'master' }} + uses: peaceiris/actions-gh-pages@v3.9.3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: gh-pages + publish_dir: ./target/doc diff --git a/.github/workflows/modules/chain-specs.yml b/.github/workflows/modules/deploy/upload-chain-specs.yml similarity index 65% rename from .github/workflows/modules/chain-specs.yml rename to .github/workflows/modules/deploy/upload-chain-specs.yml index c7e215fc9..16a3b14b4 100644 --- a/.github/workflows/modules/chain-specs.yml +++ b/.github/workflows/modules/deploy/upload-chain-specs.yml @@ -19,11 +19,6 @@ jobs: id-token: write contents: write steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - name: Install kubectl and awscli run: | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" @@ -39,25 +34,15 @@ jobs: kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default kubectl config use-context my-context - - name: Download Linux partner-chains-node artifact + - name: Download chain spec artifacts uses: actions/download-artifact@v4 with: - name: partner-chains-node-x86_64-linux-artifact-artifact - path: ./ - - - name: Generate Chain Specs - run: | - chmod +x ./partner-chains-node - source ./devnet/.envrc - ./partner-chains-node build-spec --chain local --disable-default-bootnode --raw > devnet_chain_spec.json - source ./staging/.envrc - ./partner-chains-node build-spec --chain staging --disable-default-bootnode --raw > staging_chain_spec.json + name: chain-specs - name: Update Kubernetes secret for devnet chain spec run: | SHA=${{ inputs.sha }} - TAG=${{ inputs.tag }} - SECRET_NAME="devnet-chain-spec-${SHA}-${TAG}" + SECRET_NAME="devnet-chain-spec-${SHA}" kubectl delete secret "$SECRET_NAME" --namespace=sc --ignore-not-found kubectl create secret generic "$SECRET_NAME" \ @@ -67,11 +52,29 @@ jobs: - name: Update Kubernetes secret for staging chain spec run: | SHA=${{ inputs.sha }} - TAG=${{ inputs.tag }} - SECRET_NAME="staging-chain-spec-${SHA}-${TAG}" + SECRET_NAME="staging-chain-spec-${SHA}" kubectl delete secret "$SECRET_NAME" --namespace=staging --ignore-not-found kubectl create secret generic "$SECRET_NAME" \ --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json \ --namespace=staging - \ No newline at end of file + + - name: Update Kubernetes secret for staging-preview chain spec + run: | + SHA=${{ inputs.sha }} + SECRET_NAME="staging-preview-chain-spec-${SHA}" + + kubectl delete secret "$SECRET_NAME" --namespace=staging-preview --ignore-not-found + kubectl create secret generic "$SECRET_NAME" \ + --from-file=staging_preview_chain_spec.json=./artifacts/staging_preview_chain_spec.json \ + --namespace=staging-preview + + - name: Update Kubernetes secret for staging-preprod chain spec + run: | + SHA=${{ inputs.sha }} + SECRET_NAME="staging-preprod-chain-spec-${SHA}" + + kubectl delete secret "$SECRET_NAME" --namespace=staging-preprod --ignore-not-found + kubectl create secret generic "$SECRET_NAME" \ + --from-file=staging_preprod_chain_spec.json=./artifacts/staging_preprod_chain_spec.json \ + --namespace=staging-preprod \ No newline at end of file diff --git a/.github/workflows/modules/release/create-draft-release.yml b/.github/workflows/modules/release/create-draft-release.yml new file mode 100644 index 000000000..4809c8158 --- /dev/null +++ b/.github/workflows/modules/release/create-draft-release.yml @@ -0,0 +1,111 @@ +name: Create Draft Release + +on: + workflow_call: + inputs: + tag: + description: 'Tag for the release' + required: true + type: string + secrets: + GITHUB_TOKEN: + required: true + +jobs: + pre-release-candidate: + runs-on: ubuntu-latest + steps: + - name: Set filename variables + id: set-filenames + run: | + echo "PARTNER_CHAINS_CLI_X86_64_LINUX=partner-chains-cli-${{ inputs.tag }}-x86_64-linux" >> $GITHUB_ENV + echo "PARTNER_CHAINS_NODE_X86_64_LINUX=partner-chains-node-${{ inputs.tag }}-x86_64-linux" >> $GITHUB_ENV + echo "PARTNER_CHAINS_CLI_X86_64_APPLE_DARWIN=partner-chains-cli-${{ inputs.tag }}-x86_64-apple-darwin" >> $GITHUB_ENV + echo "PARTNER_CHAINS_NODE_X86_64_APPLE_DARWIN=partner-chains-node-${{ inputs.tag }}-x86_64-apple-darwin" >> $GITHUB_ENV + echo "PARTNER_CHAINS_CLI_AARCH64_APPLE_DARWIN=partner-chains-cli-${{ inputs.tag }}-aarch64-apple-darwin" >> $GITHUB_ENV + echo "PARTNER_CHAINS_NODE_AARCH64_APPLE_DARWIN=partner-chains-node-${{ inputs.tag }}-aarch64-apple-darwin" >> $GITHUB_ENV + + - name: Download Linux CLI artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.PARTNER_CHAINS_CLI_X86_64_LINUX }} + path: artifact-linux/ + + - name: Download Linux NODE artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.PARTNER_CHAINS_NODE_X86_64_LINUX }} + path: artifact-linux/ + + - name: Download macOS x86_64 CLI artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.PARTNER_CHAINS_CLI_X86_64_APPLE_DARWIN }} + path: artifact-macos-x86_64/ + + - name: Download macOS x86_64 NODE artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.PARTNER_CHAINS_NODE_X86_64_APPLE_DARWIN }} + path: artifact-macos-x86_64/ + + - name: Download macOS ARM64 CLI artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.PARTNER_CHAINS_CLI_AARCH64_APPLE_DARWIN }} + path: artifact-macos-arm64/ + + - name: Download macOS ARM64 NODE artifact + uses: actions/download-artifact@v4 + with: + name: ${{ env.PARTNER_CHAINS_NODE_AARCH64_APPLE_DARWIN }} + path: artifact-macos-arm64/ + + - name: Check if release already exists + id: check_release + run: | + tag="${{ inputs.tag }}" + release_response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/releases/tags/$tag") + if echo "$release_response" | grep -q '"message": "Not Found"'; then + echo "release_exists=false" >> $GITHUB_ENV + echo "::set-output name=release_exists::false" + else + echo "release_exists=true" >> $GITHUB_ENV + echo "::set-output name=release_exists::true" + echo "release_id=$(echo $release_response | jq -r .id)" >> $GITHUB_ENV + echo "::set-output name=release_id::$(echo $release_response | jq -r .id)" + fi + + - name: Create draft release + id: create_release + if: ${{ steps.check_release.outputs.release_exists == 'false' }} + run: | + tag="${{ inputs.tag }}" + release_response=$(curl -s -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -d '{"tag_name": "'$tag'", "name": "'$tag'", "body": "Draft release for '$tag'", "draft": true}' \ + "https://api.github.com/repos/${{ github.repository }}/releases") + echo "release_id=$(echo $release_response | jq -r .id)" >> $GITHUB_ENV + echo "::set-output name=release_id::$(echo $release_response | jq -r .id)" + + - name: Upload artifacts to release + if: ${{ steps.check_release.outputs.release_exists == 'true' || steps.create_release.outputs.release_id != '' }} + run: | + release_id="${{ steps.create_release.outputs.release_id }}" + if [ -z "$release_id" ]; then + release_id="${{ steps.check_release.outputs.release_id }}" + fi + + for artifact in "artifact-linux/${{ env.PARTNER_CHAINS_CLI_X86_64_LINUX }}" \ + "artifact-linux/${{ env.PARTNER_CHAINS_NODE_X86_64_LINUX }}" \ + "artifact-macos-x86_64/${{ env.PARTNER_CHAINS_CLI_X86_64_APPLE_DARWIN }}" \ + "artifact-macos-x86_64/${{ env.PARTNER_CHAINS_NODE_X86_64_APPLE_DARWIN }}" \ + "artifact-macos-arm64/${{ env.PARTNER_CHAINS_CLI_AARCH64_APPLE_DARWIN }}" \ + "artifact-macos-arm64/${{ env.PARTNER_CHAINS_NODE_AARCH64_APPLE_DARWIN }}"; do + chmod +x "$artifact" + curl -s -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Content-Type: application/octet-stream" \ + --data-binary @"$artifact" \ + "https://uploads.github.com/repos/${{ github.repository }}/releases/$release_id/assets?name=$(basename $artifact)" + done diff --git a/.github/workflows/modules/release/publish-release.yml b/.github/workflows/modules/release/publish-release.yml new file mode 100644 index 000000000..99ea9a034 --- /dev/null +++ b/.github/workflows/modules/release/publish-release.yml @@ -0,0 +1,40 @@ +name: Publish Release + +on: + workflow_call: + inputs: + tag: + description: 'Tag for the release' + required: true + type: string + secrets: + GITHUB_TOKEN: + required: true + +jobs: + publish-release: + runs-on: ubuntu-latest + steps: + - name: Check if release exists + id: check_release + run: | + tag="${{ inputs.tag }}" + release_response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + "https://api.github.com/repos/${{ github.repository }}/releases/tags/$tag") + if echo "$release_response" | grep -q '"message": "Not Found"'; then + echo "release_exists=false" >> $GITHUB_ENV + echo "::set-output name=release_exists::false" + else + echo "release_exists=true" >> $GITHUB_ENV + echo "::set-output name=release_exists::true" + echo "release_id=$(echo $release_response | jq -r .id)" >> $GITHUB_ENV + echo "::set-output name=release_id::$(echo $release_response | jq -r .id)" + fi + + - name: Publish release + if: ${{ steps.check_release.outputs.release_exists == 'true' }} + run: | + release_id="${{ steps.check_release.outputs.release_id }}" + curl -s -X PATCH -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -d '{"draft": false}' \ + "https://api.github.com/repos/${{ github.repository }}/releases/$release_id" diff --git a/.github/workflows/modules/tests/e2e.yml b/.github/workflows/modules/tests/argocd-tests.yml similarity index 100% rename from .github/workflows/modules/tests/e2e.yml rename to .github/workflows/modules/tests/argocd-tests.yml diff --git a/.github/workflows/modules/tests/devshell-tests.tml b/.github/workflows/modules/tests/devshell-tests.tml new file mode 100644 index 000000000..0a8a79766 --- /dev/null +++ b/.github/workflows/modules/tests/devshell-tests.tml @@ -0,0 +1,55 @@ +name: Nix Devshells Tests + +on: + workflow_call: + secrets: + AWS_ROLE_ARN: + required: true + NIX_SIGNING_KEY: + required: true + AWS_ACCESS_KEY_ID: + required: false + AWS_SECRET_ACCESS_KEY: + required: false + AWS_SESSION_TOKEN: + required: false + +jobs: + build-and-test: + permissions: + id-token: write + contents: read + strategy: + matrix: + os: [nixos, macos] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Update flake.lock with calling PR + if: ${{ github.event_name == 'repository_dispatch' }} + run: | + nix flake lock --update-input trustless-sidechain \ + --override-input trustless-sidechain \ + github:input-output-hk/partner-chains-smart-contracts/${{ github.event.client_payload.ref }} + + - name: Acquire AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Add signing key for nix + run: echo "${{ secrets.NIX_SIGNING_KEY }}" > "${{ runner.temp }}/nix-key" + + - name: Run nixci to build/test all outputs + run: | + nix run github:srid/nixci -- -v build -- --fallback > /tmp/outputs + + - name: Copy nix scopes to nix cache + run: | + nix-store --stdin -q --deriver < /tmp/outputs | nix-store --stdin -qR --include-outputs \ + | nix copy --stdin --to \ + "s3://cache.sc.iog.io?secret-key=${{ runner.temp }}/nix-key®ion=$AWS_DEFAULT_REGION" \ + && rm /tmp/outputs diff --git a/.github/workflows/modules/tests/staging-preprod-tests.yml b/.github/workflows/modules/tests/staging-preprod-tests.yml new file mode 100644 index 000000000..e69de29bb diff --git a/.github/workflows/modules/tests/staging-preview-tests.yml b/.github/workflows/modules/tests/staging-preview-tests.yml new file mode 100644 index 000000000..e69de29bb