From 837fb7dab999de9f4d8cf7762137c469c208905b Mon Sep 17 00:00:00 2001 From: auricom <27022259+auricom@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:42:31 +0200 Subject: [PATCH] wip --- .github/scripts/prepare-matrices.py | 2 + ...o-dependencies.sh => update-go-modules.sh} | 0 ...plication.yaml => build-applications.yaml} | 83 +++++++++++-------- .github/workflows/nightlies-scheduled.yaml | 42 ++++++++++ .github/workflows/simple-checks.yaml | 38 +++++++++ apps/cosmos-sdk-comet/metadata.yaml | 14 ++++ apps/cosmos-sdk/metadata.yaml | 3 +- apps/ibc-go/metadata.yaml | 3 +- metadata.rules.cue | 19 +++++ 9 files changed, 168 insertions(+), 36 deletions(-) rename .github/scripts/{update-go-dependencies.sh => update-go-modules.sh} (100%) rename .github/workflows/{build-application.yaml => build-applications.yaml} (78%) create mode 100644 .github/workflows/nightlies-scheduled.yaml create mode 100644 .github/workflows/simple-checks.yaml create mode 100644 apps/cosmos-sdk-comet/metadata.yaml create mode 100644 metadata.rules.cue diff --git a/.github/scripts/prepare-matrices.py b/.github/scripts/prepare-matrices.py index 8fa4ee4..3643e7d 100644 --- a/.github/scripts/prepare-matrices.py +++ b/.github/scripts/prepare-matrices.py @@ -44,6 +44,7 @@ def get_app_metadata(subdir, meta, forRelease=False, channels=None): toBuild["path"] = meta["path"] toBuild["branch"] = channel["branch"] toBuild["update_modules"] = channel["update_modules"] + toBuild["build_command"] = meta["build_command"] toBuild["build_artifacts"] = meta["build_artifacts"] # Container Tags @@ -84,6 +85,7 @@ def get_app_metadata(subdir, meta, forRelease=False, channels=None): platformToBuild["tests_enabled"] = channel["tests"]["enabled"] and platform in TESTABLE_PLATFORMS + platformToBuild["build_command"] = toBuild["build_command"] platformToBuild["build_artifacts"] = meta["build_artifacts"] platformToBuild["update_modules"] = channel["update_modules"] diff --git a/.github/scripts/update-go-dependencies.sh b/.github/scripts/update-go-modules.sh similarity index 100% rename from .github/scripts/update-go-dependencies.sh rename to .github/scripts/update-go-modules.sh diff --git a/.github/workflows/build-application.yaml b/.github/workflows/build-applications.yaml similarity index 78% rename from .github/workflows/build-application.yaml rename to .github/workflows/build-applications.yaml index 7c1cd2e..1d6ed02 100644 --- a/.github/workflows/build-application.yaml +++ b/.github/workflows/build-applications.yaml @@ -11,7 +11,7 @@ on: required: false type: boolean default: false - pushImages: + publishArtifacts: required: false type: boolean default: false @@ -19,9 +19,9 @@ on: required: false default: false type: boolean + push: jobs: - prepare: name: Prepare to Build runs-on: ubuntu-latest @@ -32,12 +32,12 @@ jobs: shell: bash run: echo "LOWERCASE_REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV - - name: Generate Token - uses: actions/create-github-app-token@v1 - id: app-token - with: - app-id: "${{ secrets.BOT_APP_ID }}" - private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" + # - name: Generate Token + # uses: actions/create-github-app-token@v1 + # id: app-token + # with: + # app-id: "${{ secrets.BOT_APP_ID }}" + # private-key: "${{ secrets.BOT_APP_PRIVATE_KEY }}" - name: Checkout uses: actions/checkout@v4 @@ -60,43 +60,42 @@ jobs: TOKEN: ${{ steps.app-token.outputs.token }} shell: bash run: | + set -x if [[ -z "${{ inputs.appsToBuild }}" ]]; then - matrices=$(python ./.github/scripts/prepare-matrices.py "all" "${{ inputs.pushImages }}") + matrices=$(python ./.github/scripts/prepare-matrices.py "all" "${{ inputs.publishArtifacts }}") else - if [[ -z "${{ inputs.channelsToBuild }}" ]]; then - matrices=$(python ./.github/scripts/prepare-matrices.py "${{ inputs.appsToBuild }}" "${{ inputs.pushImages }}") - else - matrices=$(python ./.github/scripts/prepare-matrices.py "${{ inputs.appsToBuild }}" "${{ inputs.pushImages }}" "${{ inputs.channelsToBuild }}") - fi + matrices=$(python ./.github/scripts/prepare-matrices.py "${{ inputs.appsToBuild }}" "${{ inputs.publishArtifacts }}") fi echo "matrices=${matrices}" >> $GITHUB_OUTPUT echo "${matrices}" - - build-app: - runs-on: ubuntu-latest + build-platform-app: + name: Build/Test ${{ matrix.app.name }} (${{ matrix.app.platform }}) needs: prepare - if: ${{ toJSON(fromJSON(needs.prepare.outputs.matrices).app) != '[]' && toJSON(fromJSON(needs.prepare.outputs.matrices).app) != '' }} + runs-on: ubuntu-latest + if: ${{ toJSON(fromJSON(needs.prepare.outputs.matrices).appPlatforms) != '[]' && toJSON(fromJSON(needs.prepare.outputs.matrices).appPlatforms) != '' }} outputs: date: ${{ steps.archive.outputs.date }} strategy: fail-fast: false matrix: - go-arch: [amd64, arm64] - app: "${{ fromJSON(needs.prepare.outputs.matrices).app }}" + app: "${{ fromJSON(needs.prepare.outputs.matrices).appPlatforms }}" + permissions: + contents: read + packages: write steps: - name: Log Matrix Input shell: bash run: | cat << EOF - ${{ toJSON(matrix.image)}} + ${{ toJSON(matrix.app)}} EOF - uses: actions/checkout@v4 with: - repository: cosmos/cosmos-sdk - ref: ${{ matrix.major-version == 'main' && 'main' || format('refs/heads/release/{0}', matrix.major-version) }} + repository: ${{ matrix.app.repository }} + ref: ${{ matrix.app.branch }} token: ${{ github.token }} - path: cosmos-sdk + path: ${{ matrix.app.repository }} - uses: actions/checkout@v4 with: @@ -107,26 +106,42 @@ jobs: go-version: "1.23" check-latest: true - - name: Download go.mod and go.sum - uses: actions/download-artifact@v4 - if: ${{ inputs.depsUpdate }} - with: - name: go-files-${{ matrix.major-version }} - path: cosmos-sdk/simapp + - name: Update go modules + # Disable dependencies updates for v0.52.x + if: matrix.major-version != 'v0.52.x' + run: | + set -x + ls -l + ls -l ../ + ls -l ../../ + cd ${{ matrix.app.repository }}/${{ matrix.app.path }} + COSMOSSDK_BRANCH=${{ matrix.app.branch }} ../../nightly-stack/.github/scripts/update-go-modules.sh + + - name: show output of modified go.sum and go.mod + run: | + echo "############" + echo "# go.mod" + echo "############" + cat ${{ matrix.app.repository }}/${{ matrix.app.path }}/go.mod + echo -e "\n\n" + echo "############" + echo "# go.mod" + echo "############" + cat ${{ matrix.app.repository }}/${{ matrix.app.path }}/go.sum - name: Create application binary id: build run: | - cd cosmos-sdk/simapp + cd ${{ matrix.app.repository }}/${{ matrix.app.path }} go mod tidy cd ../ - GOARCH=${{ matrix.go-arch }} make install + GOARCH=${{ matrix.app.target_arch }} ${{ matrix.app.build_command }} echo "gobin=$(go env GOPATH)/bin" >> $GITHUB_OUTPUT - name: Run and monitor application shell: bash # no arm64 runners as of now : https://github.com/orgs/community/discussions/19197 - if: ${{ matrix.go-arch == 'amd64' }} + if: ${{ matrix.app.target_arch == 'amd64' }} run: | set -oue pipefail set -x @@ -189,7 +204,7 @@ jobs: with: registry: ghcr.io username: "${{ github.actor }}" - password: "${{ secrets.GI THUB_TOKEN }}" + password: "${{ secrets.GITHUB_TOKEN }}" - name: Setup Goss if: ${{ matrix.go-arch == 'amd64' }} diff --git a/.github/workflows/nightlies-scheduled.yaml b/.github/workflows/nightlies-scheduled.yaml new file mode 100644 index 0000000..df0f1a0 --- /dev/null +++ b/.github/workflows/nightlies-scheduled.yaml @@ -0,0 +1,42 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: Scheduled Nightlies + +concurrency: + group: nightlies + cancel-in-progress: false + +on: + workflow_dispatch: + inputs: + appsToBuild: + description: App(s) to build + required: false + type: string + default: all + schedule: + - cron: "0 0 * * *" + +jobs: + simple-checks: + name: Simple Checks + uses: .github/workflows/simple-checks.yaml + + build-images: + name: Build Images + needs: simple-checks + uses: .github/workflows/build-applications.yaml + secrets: inherit + permissions: + contents: read + packages: write + with: + appsToBuild: ${{ inputs.appsToBuild }} + publishArtifacts: true + sendNotifications: true + + # render-readme: + # name: Render Readme + # needs: build-images + # uses: ./.github/workflows/render-readme.yaml + # secrets: inherit \ No newline at end of file diff --git a/.github/workflows/simple-checks.yaml b/.github/workflows/simple-checks.yaml new file mode 100644 index 0000000..467e01d --- /dev/null +++ b/.github/workflows/simple-checks.yaml @@ -0,0 +1,38 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: "Simple Checks" + +on: + workflow_call: + +jobs: + metadata-validation: + name: Validate App Metadata + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Get Changed Files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files_yaml: | + cue: + - metadata.rules.cue + - apps/**/metadata.json + - apps/**/metadata.yaml + + - name: Setup CUE + if: ${{ steps.changed-files.outputs.cue_any_changed == 'true' }} + uses: cue-lang/setup-cue@v1.0.1 + + # Run against all files to ensure they are tested if the cue schema is changed. + - name: Validate image metadata + if: ${{ steps.changed-files.outputs.cue_any_changed == 'true' }} + shell: bash + run: | + find ./apps/ -name metadata.json | xargs -I {} cue vet --schema '#Spec' {} ./metadata.rules.cue + find ./apps/ -name metadata.yaml | xargs -I {} cue vet --schema '#Spec' {} ./metadata.rules.cue \ No newline at end of file diff --git a/apps/cosmos-sdk-comet/metadata.yaml b/apps/cosmos-sdk-comet/metadata.yaml new file mode 100644 index 0000000..3994935 --- /dev/null +++ b/apps/cosmos-sdk-comet/metadata.yaml @@ -0,0 +1,14 @@ +--- +app: cosmos-sdk +repository: cosmos/cosmos-sdk +path: simapp +build_command: make install +build_artifacts: true +channels: + - name: main + platforms: ["linux/amd64","linux/arm64"] + branch: main + update_modules: false + tests: + enabled: true + type: web diff --git a/apps/cosmos-sdk/metadata.yaml b/apps/cosmos-sdk/metadata.yaml index b6305ed..943b4ec 100644 --- a/apps/cosmos-sdk/metadata.yaml +++ b/apps/cosmos-sdk/metadata.yaml @@ -1,7 +1,8 @@ --- app: cosmos-sdk -repository: github.com/cosmos/cosmos-sdk +repository: cosmos/cosmos-sdk path: simapp +build_command: make install build_artifacts: true channels: - name: main diff --git a/apps/ibc-go/metadata.yaml b/apps/ibc-go/metadata.yaml index 21f23a5..4e029dc 100644 --- a/apps/ibc-go/metadata.yaml +++ b/apps/ibc-go/metadata.yaml @@ -1,7 +1,8 @@ --- app: ibc-go -repository: github.com/cosmos/ibc-go +repository: cosmos/ibc-go path: simapp +build_command: make build build_artifacts: true channels: - name: main diff --git a/metadata.rules.cue b/metadata.rules.cue new file mode 100644 index 0000000..b0cff90 --- /dev/null +++ b/metadata.rules.cue @@ -0,0 +1,19 @@ +#Spec: { + app: #AppName + semver?: bool + channels: [...#Channels] +} + +#Channels: { + name: #ChannelName + platforms: [...#Platforms] + stable: bool + tests: { + enabled: bool + type?: =~"^(cli|web)$" + } +} + +#AppName: string & !="" & =~"^[a-zA-Z0-9_-]+$" +#ChannelName: string & !="" & =~"^[a-zA-Z0-9._-]+$" +#Platforms: "linux/amd64" | "linux/arm64" \ No newline at end of file