diff --git a/.github/workflows/cicd.yml b/.github/workflows/cd.yml similarity index 100% rename from .github/workflows/cicd.yml rename to .github/workflows/cd.yml diff --git a/.github/workflows/earthly.yml b/.github/workflows/ci.yml similarity index 73% rename from .github/workflows/earthly.yml rename to .github/workflows/ci.yml index f10f4d253..6a533de2d 100644 --- a/.github/workflows/earthly.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,13 @@ -name: earthly ci +name: Main Workflow on: + push: + branches: + - master pull_request: types: [opened, synchronize, reopened, closed] branches: - - master + - '**' workflow_dispatch: inputs: upload: @@ -19,14 +22,22 @@ on: description: "Branch name to build" required: true default: 'master' + rustdoc: + description: "Build and publish the rustdoc" + default: false + type: boolean + repository_dispatch: + types: [test-run-pr] + env: AWS_REGION: "eu-central-1" + AWS_DEFAULT_REGION: "eu-central-1" SSH_AUTH_SOCK: /tmp/ssh_agent.sock jobs: build-and-push: runs-on: ubuntu-latest - if: github.event.action != 'closed' || github.event.pull_request.merged == true + if: ${{ github.event.action != 'closed' || github.event.pull_request.merged == true }} concurrency: group: pr-${{ github.event.pull_request.number }}-author-${{ github.event.pull_request.user.login }} cancel-in-progress: true @@ -167,22 +178,22 @@ jobs: ./staging_chain_spec.json generate-manifest: - needs: build-and-push - if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create and Push Manifest - env: - GH_TOKEN: ${{ secrets.ACTIONS_PAT }} - run: | - cd .github/workflows/modules/argocd - bash generate-manifest.sh ${{ github.sha }} - - - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) - run: sleep 720s + needs: build-and-push + if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create and Push Manifest + env: + GH_TOKEN: ${{ secrets.ACTIONS_PAT }} + run: | + cd .github/workflows/modules/argocd + bash generate-manifest.sh ${{ github.sha }} + + - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) + run: sleep 720s run-e2e-tests: needs: generate-manifest @@ -193,10 +204,10 @@ jobs: secrets: inherit teardown: - runs-on: ubuntu-latest - needs: [build-and-push, generate-manifest, run-e2e-tests] - if: always() && needs.generate-manifest.result == 'success' - steps: + runs-on: ubuntu-latest + needs: [build-and-push, generate-manifest, run-e2e-tests] + if: ${{ always() && needs.generate-manifest.result == 'success' }} + steps: - name: Checkout ArgoCD Repository uses: actions/checkout@v4 with: @@ -242,7 +253,7 @@ jobs: chain-specs: runs-on: [self-hosted, eks] needs: [build-and-push] - if: github.ref_name == 'master' || inputs.upload == true + if: ${{ github.ref_name == 'master' || inputs.upload == 'true' }} permissions: id-token: write contents: write @@ -279,3 +290,85 @@ jobs: TIMESTAMP=$(date +%Y%m%d%H%M) SHA=${{ github.sha }} kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging + + deploy-rustdoc: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.rustdoc == 'true' }} + runs-on: ubuntu-latest + steps: + - name: Install tooling + run: | + sudo apt-get install -y protobuf-compiler + protoc --version + + - name: Checkout repository + uses: actions/checkout@v3.5.3 + + - name: Add SSH key to read Substrate Repo + run: | + mkdir ~/.ssh + ssh-keyscan github.com >> ~/.ssh/known_hosts + ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null + ssh-add - <<< "${{ secrets.SUBSTRATE_REPO_SSH_KEY }}" + + - name: Rust versions + run: rustup show + + - name: Rust cache + uses: Swatinem/rust-cache@v2.6.2 + + - name: Build rustdocs + run: SKIP_WASM_BUILD=1 cargo doc --all --no-deps + + - name: Make index.html + run: echo "" > ./target/doc/index.html + + - name: Deploy documentation + if: ${{ github.ref_name == 'master' }} + uses: peaceiris/actions-gh-pages@v3.9.3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_branch: gh-pages + publish_dir: ./target/doc + + build-and-test: + permissions: + id-token: write + contents: read + strategy: + matrix: + os: [nixos, macos] + runs-on: + - self-hosted + - ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ github.token }} + + - name: Update flake.lock with calling PR + if: ${{ github.event_name == 'repository_dispatch' }} + run: | + nix flake lock --update-input trustless-sidechain \ + --override-input trustless-sidechain \ + github:input-output-hk/partner-chains-smart-contracts/${{ github.event.client_payload.ref }} + + - name: Acquire AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + - name: Add signing key for nix + run: echo "${{ secrets.NIX_SIGNING_KEY }}" > "${{ runner.temp }}/nix-key" + + - name: Run nixci to build/test all outputs + run: | + nix run github:srid/nixci -- -v build -- --fallback > /tmp/outputs + + - name: Copy nix scopes to nix cache + run: | + nix-store --stdin -q --deriver < /tmp/outputs | nix-store --stdin -qR --include-outputs \ + | nix copy --stdin --to \ + "s3://cache.sc.iog.io?secret-key=${{ runner.temp }}/nix-key®ion=$AWS_DEFAULT_REGION" \ + && rm /tmp/outputs diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml deleted file mode 100644 index abd14af6b..000000000 --- a/.github/workflows/nix.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: "Nix Devshells CI" -on: - pull_request: - push: - branches: - - master - workflow_dispatch: - repository_dispatch: - types: [test-run-pr] - -env: - AWS_DEFAULT_REGION: eu-central-1 - -jobs: - build-and-test: - permissions: - id-token: write - contents: read - strategy: - matrix: - os: [nixos, macos] - runs-on: - - self-hosted - - ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - token: ${{ github.token }} - - name: Update flake.lock with calling PR - if: github.event_name == 'repository_dispatch' - run: | - nix flake lock --update-input trustless-sidechain \ - --override-input trustless-sidechain \ - github:input-output-hk/partner-chains-smart-contracts/${{ github.event.client_payload.ref }} - - name: Acquire AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN_ }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} - - - name: Add signing key for nix - run: echo "${{ secrets.NIX_SIGNING_KEY }}" > "${{ runner.temp }}/nix-key" - - name: Run nixci to build/test all outputs - run: | - nix run github:srid/nixci -- -v build -- --fallback > /tmp/outputs - - name: Copy nix scopes to nix cache - run: | - nix-store --stdin -q --deriver < /tmp/outputs | nix-store --stdin -qR --include-outputs \ - | nix copy --stdin --to \ - "s3://cache.sc.iog.io?secret-key=${{ runner.temp }}/nix-key®ion=$AWS_DEFAULT_REGION" \ - && rm /tmp/outputs diff --git a/.github/workflows/not-earthly.yml b/.github/workflows/not-earthly.yml deleted file mode 100644 index 026c69609..000000000 --- a/.github/workflows/not-earthly.yml +++ /dev/null @@ -1,210 +0,0 @@ -name: Not earthly ci - -on: - pull_request: - types: [opened, synchronize, reopened, closed] - branches: - - master - workflow_dispatch: - inputs: - upload: - description: "Upload the container to our registry" - default: false - type: boolean - -env: - AWS_REGION: "eu-central-1" - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - -jobs: - build-and-push-without-earthly: - runs-on: ubuntu-latest - if: github.event.action != 'closed' || github.event.pull_request.merged == true - permissions: - id-token: write - contents: write - steps: - - name: Checkout partner-chains - uses: actions/checkout@v4 - - - name: Acquire AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN_SECRET }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to ECR - uses: docker/login-action@v3 - with: - registry: ${{ secrets.ECR_REGISTRY_SECRET }} - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env - rustup target add x86_64-unknown-linux-gnu - - - name: Build partner-chains-node - run: | - cargo build -p partner-chains-node --locked --release --target x86_64-unknown-linux-gnu - chmod +x target/x86_64-unknown-linux-gnu/release/partner-chains-node - cp target/x86_64-unknown-linux-gnu/release/partner-chains-node ./partner-chains-node - - - name: Run tests - run: cargo test --locked --release --target x86_64-unknown-linux-gnu - - - name: Generate Chain Specs - run: | - chmod +x ./partner-chains-node - source ./devnet/.envrc - ./partner-chains-node build-spec --chain local --disable-default-bootnode --raw > devnet_chain_spec.json - source ./staging/.envrc - ./partner-chains-node build-spec --chain staging --disable-default-bootnode --raw > staging_chain_spec.json - - - name: Create and Configure Docker Container - id: create-container - run: | - container_id=$(docker run -d debian:bullseye-slim sleep infinity) - echo "container_id=$container_id" >> $GITHUB_ENV - docker exec $container_id useradd -m -u 1000 -U -s /bin/sh -d /substrate substrate - docker exec $container_id mkdir -p /data /substrate/.local/share/partner-chains-node - docker exec $container_id chown -R substrate:substrate /data /substrate - docker exec $container_id rm -rf /usr/bin/apt* /usr/bin/dpkg* - docker exec $container_id ln -s /data /substrate/.local/share/partner-chains-node - docker cp ./partner-chains-node $container_id:/usr/local/bin/partner-chains-node - docker commit --change='EXPOSE 30333 9615 9933 9944' --change='ENTRYPOINT ["/usr/local/bin/partner-chains-node"]' $container_id substrate-node:${{ github.sha }} - - - name: Push to ECR - run: | - docker tag substrate-node:${{ github.sha }} ${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node:${{ github.sha }} - docker push ${{ secrets.ECR_REGISTRY_SECRET }}/substrate-node:${{ github.sha }} - - - name: Cleanup Docker Container - if: always() - run: | - docker rm -f ${{ env.container_id }} - - - name: Upload chain spec artifacts - uses: actions/upload-artifact@v4 - if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} - with: - name: chain-specs - path: | - ./devnet_chain_spec.json - ./staging_chain_spec.json - - generate-manifest: - needs: build-and-push-without-earthly - if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create and Push Manifest - env: - GH_TOKEN: ${{ secrets.ACTIONS_PAT }} - run: | - cd .github/workflows/argocd - bash generate-manifest.sh ${{ github.sha }} - - - name: Wait for 12 minutes (ArgoCD refresh interval is 3 minutes + 1 minute to build + 8 minutes for node to start producing blocks) - run: sleep 720s - - run-e2e-tests: - needs: generate-manifest - uses: ./.github/workflows/e2e.yml - with: - node-host: sha-${{ github.sha }}-service.integration-testing.svc.cluster.local - node-port: 9933 - secrets: inherit - - teardown: - runs-on: ubuntu-latest - needs: [build-and-push-without-earthly, generate-manifest, run-e2e-tests] - if: always() && needs.generate-manifest.result == 'success' - steps: - - name: Checkout ArgoCD Repository - uses: actions/checkout@v4 - with: - repository: input-output-hk/sidechains-argocd - token: ${{ secrets.ACTIONS_PAT }} - path: sidechains-argocd - - - name: Delete Ephemeral Environment Files - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.ACTIONS_PAT }} - script: | - const fs = require('fs'); - const path = require('path'); - - const directory = 'sidechains-argocd/integration-testing'; - const files = fs.readdirSync(directory); - - for (const file of files) { - if (file.startsWith('manifest-sha-')) { - console.log(`Deleting file: ${file}`); - - // Fetch the SHA of the file - const shaResponse = await github.rest.repos.getContent({ - owner: 'input-output-hk', - repo: 'sidechains-argocd', - path: `integration-testing/${file}`, - }); - const sha = shaResponse.data.sha; - - // GitHub API request to delete the file - await github.rest.repos.deleteFile({ - owner: 'input-output-hk', - repo: 'sidechains-argocd', - path: `integration-testing/${file}`, - message: `ci: Tear down integration-testing environment for SHA #${file.split('-').pop().split('.')[0]}`, - sha: sha, - branch: 'main' - }); - } - } - - chain-specs: - runs-on: [self-hosted, eks] - needs: [build-and-push-without-earthly] - if: ${{ github.event.pull_request.merged == true && !contains(github.event.pull_request.labels.*.name, 'ci-off') }} - permissions: - id-token: write - contents: write - steps: - - name: Install kubectl and awscli - run: | - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" - chmod +x ./kubectl - sudo mv ./kubectl /usr/local/bin/kubectl - sudo apt update && sudo apt install -y awscli - - - name: Configure kubectl - run: | - echo "${{ secrets.kubeconfig_base64 }}" | base64 --decode > ${{ runner.temp }}/kubeconfig.yaml - kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true - kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} - kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default - kubectl config use-context my-context - - - name: Download chain spec artifacts - uses: actions/download-artifact@v4 - with: - name: chain-specs - path: ./artifacts - - - name: Update Kubernetes secret for devnet chain spec - run: | - TIMESTAMP=$(date +%Y%m%d%H%M) - SHA=${{ github.sha }} - kubectl create secret generic "devnet-chain-spec-${TIMESTAMP}-${SHA}" --from-file=devnet_chain_spec.json=./artifacts/devnet_chain_spec.json --namespace=sc - - - name: Update Kubernetes secret for staging chain spec - run: | - TIMESTAMP=$(date +%Y%m%d%H%M) - SHA=${{ github.sha }} - kubectl create secret generic "staging-chain-spec-${TIMESTAMP}-${SHA}" --from-file=staging_chain_spec.json=./artifacts/staging_chain_spec.json --namespace=staging diff --git a/.github/workflows/publish-doc.yml b/.github/workflows/publish-doc.yml deleted file mode 100644 index 2ab7fb521..000000000 --- a/.github/workflows/publish-doc.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Publish Documentation - -on: - - push: - branches: - - master - pull_request: - branches: - - master - workflow_dispatch: - inputs: - rustdoc: - description: "Build and publish the rustdoc" - default: false - type: boolean - -env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - -jobs: - deploy-rustdoc: - if: github.event_name != 'workflow_dispatch' || inputs.rustdoc - name: Deploy Rust Docs - runs-on: ubuntu-latest - - steps: - - name: Install tooling - run: | - sudo apt-get install -y protobuf-compiler - protoc --version - - - name: Checkout repository - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 - - - name: Add SSH key to read Substrate Repo - run: | - mkdir ~/.ssh - ssh-keyscan github.com >> ~/.ssh/known_hosts - ssh-agent -a "$SSH_AUTH_SOCK" > /dev/null - ssh-add - <<< "${{ secrets.SUBSTRATE_REPO_SSH_KEY }}" - - # With rustup's nice new toml format, we just need to run rustup show to install the toolchain - # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659 - - name: Rust versions - run: rustup show - - - name: Rust cache - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # v2.6.2 - - - name: Build rustdocs - run: SKIP_WASM_BUILD=1 cargo doc --all --no-deps - - - name: Make index.html - run: echo "" > ./target/doc/index.html - - - name: Deploy documentation - if: github.ref_name == 'master' - uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_branch: gh-pages - publish_dir: ./target/doc