Generate catalog using workflow matrix

For each repository+type tuple from `externals.yaml`, generate a job who's responsible to generate that part of the catalog only. Signed-off-by: Vincent Demeester <[email protected]>
openshift-pipelines · Nov 27, 2023 · 9507d15 · 9507d15
1 parent 837e9ea
commit 9507d15
Show file tree

Hide file tree

Showing 9 changed files with 401 additions and 126 deletions.
diff --git a/.github/workflows/generate-catalogs.yaml b/.github/workflows/generate-catalogs.yaml
@@ -1,5 +1,5 @@
 ---
-# This workflow generates the experimental catalog and pushes it to
+# This workflow generates the stable catalog and pushes it to
 # the publish branch.
 name: generate-catalogs
 
@@ -9,154 +9,64 @@ on:
   workflow_dispatch: # allow manual triggering
 
 jobs:
-  generate-experimental-catalog:
+
+  catalog-matrix:
     runs-on: ubuntu-latest
-    if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
+    # if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/go
-      - name: Generate experimental catalog
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - id: set-matrix
+        name: Generate catalog matrix
         run: |
-          go run ./cmd/catalog-cd generate-catalog --config ./experimental/externals.yaml experimental
-      - uses: actions/upload-artifact@v3
-        with:
-          name: experimental-catalog-artifact
-          path: experimental/
-          retention-days: 3 # We don't need to keep them for long
+          set -Eeu
+          echo "matrix=$(go run ./cmd/catalog-cd catalog externals --config ./externals.yaml)" >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
 
   generate-catalog:
+    needs: catalog-matrix
     runs-on: ubuntu-latest
-    if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
+    # if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
+    strategy:
+      matrix: ${{fromJSON(needs.catalog-matrix.outputs.matrix)}}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/go
-
-      - name: Generate catalog
+      - name: ${{ matrix.type }} catalog for ${{ matrix.name }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           mkdir -p stable/tasks stable/pipelines
           cp -fR tasks/* stable/tasks
           cp -fR pipelines/* stable/pipelines
-          go run ./cmd/catalog-cd generate-catalog --config ./externals.yaml stable
-      - uses: actions/upload-artifact@v3
-        with:
-          name: stable-catalog-artifact
-          path: stable/
-          retention-days: 3 # We don't need to keep them for long
-
-  publish-experimental-catalog:
-    runs-on: ubuntu-latest
-    if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
-    needs:
-      - generate-experimental-catalog
-    steps:
-      - name: Checkout "p" publish branch
-        uses: actions/checkout@v4
-        with:
-          ref: p
-      - name: Checkout "main" branch
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          path: main
-      - name: Sync main "helper code" with p
-        run: |
-          # Clean existing
-          rm -fR automation Makefile .github
-          # Copy from main
-          cp -fR main/{automation,Makefile} .
-          mkdir -p .github/workflows
-          mkdir -p .github/workflows .github/actions
-          cp -fR main/.github/workflows/test* .github/workflows
-          cp -fR main/.github/actions/* .github/actions
-      - name: Clean old "generated" catalogs
-        run: |
-          # Remove experimental resources
-          rm -fR experimental/{tasks,pipelines}/*
-      - name: Fetch generated experimental catalog
-        uses: actions/download-artifact@v3
-        with:
-          name: experimental-catalog-artifact
-          path: experimental
-      - name: Add experimental catalog to publish branch
-        run: |
-          # Clean main from here before the status
-          rm -fR main
-          git config user.name github-actions
-          git config user.email [email protected]
-          CHANGES=$(git status -s)
-          if [[ -n "${CHANGES}" ]]; then
-            git status
-            git add experimental Makefile automation .github
-            git commit -m "Auto-update experimental tekton resources"
-            git pull --rebase --autostash
-          else
-            echo "No changes for the experimental catalog"
-          fi
-      - name: Create PR
-        id: create-pr
-        uses: peter-evans/create-pull-request@v5
-        with:
-          base: p
-          branch: update-experimental
-          delete-branch: true
-          add-paths: README.md # This is here just to force it to not commit anything
-          title: Experimental catalog changes
-          assignees: tekton-ecosystem-team
-          labels: approved, lgtm, ok-to-test # Automatically approved :)
-          # committer: ${{ env.GIT_COMMITTER_NAME }} ${{ env.GIT_COMMITTER_EMAIL }}
-          # author: ${{ env.GIT_AUTHOR_NAME }} ${{ env.GIT_AUTHOR_EMAIL }}
-          # body: ${{ steps.pr_body.outputs.content }} # TODO(vdemeester) Write git status from the artifact
-
-
-  publish-catalog:
-    runs-on: ubuntu-latest
-    if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
-    needs:
-      - generate-catalog
-    steps:
+          go run ./cmd/catalog-cd catalog generate-from \
+                 --name ${{ matrix.name }} \
+                 --url ${{ matrix.url }} \
+                 --type ${{ matrix.type }} \
+                 --ignore-versions "${{ matrix.ignoreVersions }}" \
+                 stable
       - name: Checkout "p" publish branch
         uses: actions/checkout@v4
         with:
+          repository: openshift-pipelines/tektoncd-catalog
           ref: p
-      - name: Checkout "main" branch
-        uses: actions/checkout@v4
-        with:
-          ref: main
-          path: main
-      - name: Sync main "helper code" with p
-        run: |
-          # Clean existing
-          rm -fR automation Makefile .github
-          # Copy from main
-          cp -fR main/{automation,Makefile} .
-          mkdir -p .github/workflows .github/actions
-          cp -fR main/.github/workflows/test* .github/workflows
-          cp -fR main/.github/actions/* .github/actions
-      - name: Clean old "generated" catalogs
+          path: p
+      - name: Copy "partial" catalog ${{ matrix.name }} in publish branch
         run: |
-          # Remove "stable" resources
-          rm -fR {tasks,pipelines}/*
-      - name: Fetch generated stable catalog
-        uses: actions/download-artifact@v3
-        with:
-          name: stable-catalog-artifact
-          path: .
-      - name: Add experimental catalog to publish branch
+          cp -fRv stable/* p/
+      - name: Add ${{ matrix.type }} from ${{ matrix.name }} to publish branch
+        working-directory: p
         run: |
           # Clean main from here before the status
-          rm -fR main
           git config user.name github-actions
           git config user.email [email protected]
           CHANGES=$(git status -s)
           if [[ -n "${CHANGES}" ]]; then
             git status
             git add tasks pipelines
-            git add Makefile automation .github
-            git commit -m "Auto-update tekton resources"
+            git commit -m "${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources\n\nURL: ${{ matrix.url }}\\nIgnoredVersions: ${{ matrix.ignoreVersions }}"
             git pull --rebase --autostash
           else
             echo "No changes for the catalog"
@@ -169,9 +79,14 @@ jobs:
           branch: update-stable
           delete-branch: true
           add-paths: README.md # This is here just to force it to not commit anything
-          title: Catalog changes
+          title: "${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources"
           assignees: tekton-ecosystem-team
           labels: approved, lgtm, ok-to-test # Automatically approved :)
           # committer: ${{ env.GIT_COMMITTER_NAME }} ${{ env.GIT_COMMITTER_EMAIL }}
           # author: ${{ env.GIT_AUTHOR_NAME }} ${{ env.GIT_AUTHOR_EMAIL }}
           # body: ${{ steps.pr_body.outputs.content }} # TODO(vdemeester) Write git status from the artifact
+      - uses: actions/upload-artifact@v3 # We could ignore this completely
+        with:
+          name: ${{ matrix.name }}-${{ matrix.type }}-stable-catalog-artifact
+          path: stable/
+          retention-days: 3 # We don't need to keep them for long
diff --git a/.github/workflows/generate-experimental-catalogs.yaml b/.github/workflows/generate-experimental-catalogs.yaml
@@ -0,0 +1,92 @@
+---
+# This workflow generates the experimental catalog and pushes it to
+# the publish branch.
+name: generate-experimental-catalogs
+
+on:
+  schedule:
+    - cron: "0 */6 * * *" # every 6 hours
+  workflow_dispatch: # allow manual triggering
+
+jobs:
+
+  experimental-catalog-matrix:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/go
+      - id: set-matrix
+        name: Generate catalog matrix
+        run: |
+          set -Eeu
+          echo "matrix=$(go run ./cmd/catalog-cd catalog externals --config ./experimental/externals.yaml)" >> "$GITHUB_OUTPUT"
+          cat "$GITHUB_OUTPUT"
+
+  generate-experimental-catalog:
+    needs: experimental-catalog-matrix
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere
+    strategy:
+      matrix: ${{fromJSON(needs.experimental-catalog-matrix.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/go
+      - name: ${{ matrix.type }} catalog for ${{ matrix.name }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p experimental/tasks experimental/pipelines
+          cp -fR tasks/* experimental/tasks
+          cp -fR pipelines/* experimental/pipelines
+          go run ./cmd/catalog-cd catalog generate-from \
+                 --name ${{ matrix.name }} \
+                 --url ${{ matrix.url }} \
+                 --type ${{ matrix.type }} \
+                 --ignore-versions "${{ matrix.ignoreVersions }}" \
+                 experimental
+      - name: Checkout "p" publish branch
+        uses: actions/checkout@v4
+        with:
+          repository: openshift-pipelines/tektoncd-catalog
+          ref: p
+          path: p
+      - name: Copy "partial" catalog ${{ matrix.name }} in publish branch
+        run: |
+          cp -fRv experimental/* p/experimental/
+      - name: Add ${{ matrix.type }} from ${{ matrix.name }} to publish branch
+        working-directory: p
+        run: |
+          # Clean main from here before the status
+          git config user.name github-actions
+          git config user.email [email protected]
+          CHANGES=$(git status -s)
+          if [[ -n "${CHANGES}" ]]; then
+            git status
+            git add experimental/tasks experimental/pipelines
+            git commit -m "experimental/${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources\n\nURL: ${{ matrix.url }}\\nIgnoredVersions: ${{ matrix.ignoreVersions }}"
+            git pull --rebase --autostash
+          else
+            echo "No changes for the catalog"
+          fi
+      - name: Create PR
+        id: create-pr
+        uses: peter-evans/create-pull-request@v5
+        with:
+          base: p
+          branch: update-experimental
+          delete-branch: true
+          add-paths: README.md # This is here just to force it to not commit anything
+          title: "experimental/${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources"
+          assignees: tekton-ecosystem-team
+          labels: approved, lgtm, ok-to-test # Automatically approved :)
+          # committer: ${{ env.GIT_COMMITTER_NAME }} ${{ env.GIT_COMMITTER_EMAIL }}
+          # author: ${{ env.GIT_AUTHOR_NAME }} ${{ env.GIT_AUTHOR_EMAIL }}
+          # body: ${{ steps.pr_body.outputs.content }} # TODO(vdemeester) Write git status from the artifact
+      - uses: actions/upload-artifact@v3 # We could ignore this completely
+        with:
+          name: ${{ matrix.name }}-${{ matrix.type }}-experimental-catalog-artifact
+          path: experimental/
+          retention-days: 3 # We don't need to keep them for long
diff --git a/Makefile b/Makefile
@@ -45,6 +45,10 @@ test-e2e/kubernetes: test-e2e-kubernetes
 test-e2e-kubernetes: ## Run e2e tests on Kubernetes.
 	./automation/e2e-tests.sh kubernetes
 
+.PHONY: watch
+catalog-cd-watch: ## Watch go files and rebuild catalog-cd on changes (needs entr).
+	find . -name '*.go' | entr -r go build -v ./cmd/catalog-cd
+
 .PHONY: help
 help:
 	@grep -hE '^[ a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \

diff --git a/externals.yaml b/externals.yaml
@@ -1,6 +1,8 @@
 repositories:
 - url: https://github.com/openshift-pipelines/task-git
-  types: [ tasks pipelines ]
+  types:
+  - tasks
+  - pipelines
 - url: https://github.com/openshift-pipelines/task-containers
   types:
   - tasks

diff --git a/internal/cmd/catalog.go b/internal/cmd/catalog.go
@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"github.com/openshift-pipelines/tektoncd-catalog/internal/config"
+	"github.com/openshift-pipelines/tektoncd-catalog/internal/runner"
+	"github.com/spf13/cobra"
+)
+
+func CatalogCmd(cfg *config.Config) *cobra.Command {
+	catalogCmd := &cobra.Command{
+		Use:  "catalog",
+		Long: `Catalog management commands.`,
+	}
+
+	catalogCmd.AddCommand(runner.NewRunner(cfg, NewCatalogGenerateCmd()).Cmd())
+	catalogCmd.AddCommand(runner.NewRunner(cfg, NewCatalogGenerateFromExternalCmd()).Cmd())
+	catalogCmd.AddCommand(runner.NewRunner(cfg, NewCatalogExternalsCmd()).Cmd())
+
+	return catalogCmd
+}