From 9507d15cca383706875318cc0a08d866a325a965 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Mon, 27 Nov 2023 11:45:09 +0100 Subject: [PATCH] Generate catalog using workflow matrix For each repository+type tuple from `externals.yaml`, generate a job who's responsible to generate that part of the catalog only. Signed-off-by: Vincent Demeester --- .github/workflows/generate-catalogs.yaml | 155 ++++-------------- .../generate-experimental-catalogs.yaml | 92 +++++++++++ Makefile | 4 + externals.yaml | 4 +- internal/cmd/catalog.go | 20 +++ internal/cmd/externals.go | 125 ++++++++++++++ internal/cmd/generate-from.go | 116 +++++++++++++ internal/cmd/generate.go | 8 +- internal/cmd/root.go | 3 +- 9 files changed, 401 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/generate-experimental-catalogs.yaml create mode 100644 internal/cmd/catalog.go create mode 100644 internal/cmd/externals.go create mode 100644 internal/cmd/generate-from.go diff --git a/.github/workflows/generate-catalogs.yaml b/.github/workflows/generate-catalogs.yaml index 68524594..6411abc0 100644 --- a/.github/workflows/generate-catalogs.yaml +++ b/.github/workflows/generate-catalogs.yaml @@ -1,5 +1,5 @@ --- -# This workflow generates the experimental catalog and pushes it to +# This workflow generates the stable catalog and pushes it to # the publish branch. name: generate-catalogs @@ -9,154 +9,64 @@ on: workflow_dispatch: # allow manual triggering jobs: - generate-experimental-catalog: + + catalog-matrix: runs-on: ubuntu-latest - if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere + # if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/go - - name: Generate experimental catalog - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - id: set-matrix + name: Generate catalog matrix run: | - go run ./cmd/catalog-cd generate-catalog --config ./experimental/externals.yaml experimental - - uses: actions/upload-artifact@v3 - with: - name: experimental-catalog-artifact - path: experimental/ - retention-days: 3 # We don't need to keep them for long + set -Eeu + echo "matrix=$(go run ./cmd/catalog-cd catalog externals --config ./externals.yaml)" >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" generate-catalog: + needs: catalog-matrix runs-on: ubuntu-latest - if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere + # if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere + strategy: + matrix: ${{fromJSON(needs.catalog-matrix.outputs.matrix)}} steps: - uses: actions/checkout@v4 - uses: ./.github/actions/go - - - name: Generate catalog + - name: ${{ matrix.type }} catalog for ${{ matrix.name }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | mkdir -p stable/tasks stable/pipelines cp -fR tasks/* stable/tasks cp -fR pipelines/* stable/pipelines - go run ./cmd/catalog-cd generate-catalog --config ./externals.yaml stable - - uses: actions/upload-artifact@v3 - with: - name: stable-catalog-artifact - path: stable/ - retention-days: 3 # We don't need to keep them for long - - publish-experimental-catalog: - runs-on: ubuntu-latest - if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere - needs: - - generate-experimental-catalog - steps: - - name: Checkout "p" publish branch - uses: actions/checkout@v4 - with: - ref: p - - name: Checkout "main" branch - uses: actions/checkout@v4 - with: - ref: main - path: main - - name: Sync main "helper code" with p - run: | - # Clean existing - rm -fR automation Makefile .github - # Copy from main - cp -fR main/{automation,Makefile} . - mkdir -p .github/workflows - mkdir -p .github/workflows .github/actions - cp -fR main/.github/workflows/test* .github/workflows - cp -fR main/.github/actions/* .github/actions - - name: Clean old "generated" catalogs - run: | - # Remove experimental resources - rm -fR experimental/{tasks,pipelines}/* - - name: Fetch generated experimental catalog - uses: actions/download-artifact@v3 - with: - name: experimental-catalog-artifact - path: experimental - - name: Add experimental catalog to publish branch - run: | - # Clean main from here before the status - rm -fR main - git config user.name github-actions - git config user.email github-actions@github.com - CHANGES=$(git status -s) - if [[ -n "${CHANGES}" ]]; then - git status - git add experimental Makefile automation .github - git commit -m "Auto-update experimental tekton resources" - git pull --rebase --autostash - else - echo "No changes for the experimental catalog" - fi - - name: Create PR - id: create-pr - uses: peter-evans/create-pull-request@v5 - with: - base: p - branch: update-experimental - delete-branch: true - add-paths: README.md # This is here just to force it to not commit anything - title: Experimental catalog changes - assignees: tekton-ecosystem-team - labels: approved, lgtm, ok-to-test # Automatically approved :) - # committer: ${{ env.GIT_COMMITTER_NAME }} ${{ env.GIT_COMMITTER_EMAIL }} - # author: ${{ env.GIT_AUTHOR_NAME }} ${{ env.GIT_AUTHOR_EMAIL }} - # body: ${{ steps.pr_body.outputs.content }} # TODO(vdemeester) Write git status from the artifact - - - publish-catalog: - runs-on: ubuntu-latest - if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere - needs: - - generate-catalog - steps: + go run ./cmd/catalog-cd catalog generate-from \ + --name ${{ matrix.name }} \ + --url ${{ matrix.url }} \ + --type ${{ matrix.type }} \ + --ignore-versions "${{ matrix.ignoreVersions }}" \ + stable - name: Checkout "p" publish branch uses: actions/checkout@v4 with: + repository: openshift-pipelines/tektoncd-catalog ref: p - - name: Checkout "main" branch - uses: actions/checkout@v4 - with: - ref: main - path: main - - name: Sync main "helper code" with p - run: | - # Clean existing - rm -fR automation Makefile .github - # Copy from main - cp -fR main/{automation,Makefile} . - mkdir -p .github/workflows .github/actions - cp -fR main/.github/workflows/test* .github/workflows - cp -fR main/.github/actions/* .github/actions - - name: Clean old "generated" catalogs + path: p + - name: Copy "partial" catalog ${{ matrix.name }} in publish branch run: | - # Remove "stable" resources - rm -fR {tasks,pipelines}/* - - name: Fetch generated stable catalog - uses: actions/download-artifact@v3 - with: - name: stable-catalog-artifact - path: . - - name: Add experimental catalog to publish branch + cp -fRv stable/* p/ + - name: Add ${{ matrix.type }} from ${{ matrix.name }} to publish branch + working-directory: p run: | # Clean main from here before the status - rm -fR main git config user.name github-actions git config user.email github-actions@github.com CHANGES=$(git status -s) if [[ -n "${CHANGES}" ]]; then git status git add tasks pipelines - git add Makefile automation .github - git commit -m "Auto-update tekton resources" + git commit -m "${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources\n\nURL: ${{ matrix.url }}\\nIgnoredVersions: ${{ matrix.ignoreVersions }}" git pull --rebase --autostash else echo "No changes for the catalog" @@ -169,9 +79,14 @@ jobs: branch: update-stable delete-branch: true add-paths: README.md # This is here just to force it to not commit anything - title: Catalog changes + title: "${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources" assignees: tekton-ecosystem-team labels: approved, lgtm, ok-to-test # Automatically approved :) # committer: ${{ env.GIT_COMMITTER_NAME }} ${{ env.GIT_COMMITTER_EMAIL }} # author: ${{ env.GIT_AUTHOR_NAME }} ${{ env.GIT_AUTHOR_EMAIL }} # body: ${{ steps.pr_body.outputs.content }} # TODO(vdemeester) Write git status from the artifact + - uses: actions/upload-artifact@v3 # We could ignore this completely + with: + name: ${{ matrix.name }}-${{ matrix.type }}-stable-catalog-artifact + path: stable/ + retention-days: 3 # We don't need to keep them for long diff --git a/.github/workflows/generate-experimental-catalogs.yaml b/.github/workflows/generate-experimental-catalogs.yaml new file mode 100644 index 00000000..ca473d62 --- /dev/null +++ b/.github/workflows/generate-experimental-catalogs.yaml @@ -0,0 +1,92 @@ +--- +# This workflow generates the experimental catalog and pushes it to +# the publish branch. +name: generate-experimental-catalogs + +on: + schedule: + - cron: "0 */6 * * *" # every 6 hours + workflow_dispatch: # allow manual triggering + +jobs: + + experimental-catalog-matrix: + runs-on: ubuntu-latest + if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/go + - id: set-matrix + name: Generate catalog matrix + run: | + set -Eeu + echo "matrix=$(go run ./cmd/catalog-cd catalog externals --config ./experimental/externals.yaml)" >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + generate-experimental-catalog: + needs: experimental-catalog-matrix + runs-on: ubuntu-latest + if: github.repository_owner == 'openshift-pipelines' # do not run this elsewhere + strategy: + matrix: ${{fromJSON(needs.experimental-catalog-matrix.outputs.matrix)}} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/go + - name: ${{ matrix.type }} catalog for ${{ matrix.name }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p experimental/tasks experimental/pipelines + cp -fR tasks/* experimental/tasks + cp -fR pipelines/* experimental/pipelines + go run ./cmd/catalog-cd catalog generate-from \ + --name ${{ matrix.name }} \ + --url ${{ matrix.url }} \ + --type ${{ matrix.type }} \ + --ignore-versions "${{ matrix.ignoreVersions }}" \ + experimental + - name: Checkout "p" publish branch + uses: actions/checkout@v4 + with: + repository: openshift-pipelines/tektoncd-catalog + ref: p + path: p + - name: Copy "partial" catalog ${{ matrix.name }} in publish branch + run: | + cp -fRv experimental/* p/experimental/ + - name: Add ${{ matrix.type }} from ${{ matrix.name }} to publish branch + working-directory: p + run: | + # Clean main from here before the status + git config user.name github-actions + git config user.email github-actions@github.com + CHANGES=$(git status -s) + if [[ -n "${CHANGES}" ]]; then + git status + git add experimental/tasks experimental/pipelines + git commit -m "experimental/${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources\n\nURL: ${{ matrix.url }}\\nIgnoredVersions: ${{ matrix.ignoreVersions }}" + git pull --rebase --autostash + else + echo "No changes for the catalog" + fi + - name: Create PR + id: create-pr + uses: peter-evans/create-pull-request@v5 + with: + base: p + branch: update-experimental + delete-branch: true + add-paths: README.md # This is here just to force it to not commit anything + title: "experimental/${{ matrix.name }}: Auto-update tekton ${{ matrix.type }} resources" + assignees: tekton-ecosystem-team + labels: approved, lgtm, ok-to-test # Automatically approved :) + # committer: ${{ env.GIT_COMMITTER_NAME }} ${{ env.GIT_COMMITTER_EMAIL }} + # author: ${{ env.GIT_AUTHOR_NAME }} ${{ env.GIT_AUTHOR_EMAIL }} + # body: ${{ steps.pr_body.outputs.content }} # TODO(vdemeester) Write git status from the artifact + - uses: actions/upload-artifact@v3 # We could ignore this completely + with: + name: ${{ matrix.name }}-${{ matrix.type }}-experimental-catalog-artifact + path: experimental/ + retention-days: 3 # We don't need to keep them for long diff --git a/Makefile b/Makefile index 867e5c19..d18e16cb 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,10 @@ test-e2e/kubernetes: test-e2e-kubernetes test-e2e-kubernetes: ## Run e2e tests on Kubernetes. ./automation/e2e-tests.sh kubernetes +.PHONY: watch +catalog-cd-watch: ## Watch go files and rebuild catalog-cd on changes (needs entr). + find . -name '*.go' | entr -r go build -v ./cmd/catalog-cd + .PHONY: help help: @grep -hE '^[ a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ diff --git a/externals.yaml b/externals.yaml index 39537cfd..4affc07a 100644 --- a/externals.yaml +++ b/externals.yaml @@ -1,6 +1,8 @@ repositories: - url: https://github.com/openshift-pipelines/task-git - types: [ tasks pipelines ] + types: + - tasks + - pipelines - url: https://github.com/openshift-pipelines/task-containers types: - tasks diff --git a/internal/cmd/catalog.go b/internal/cmd/catalog.go new file mode 100644 index 00000000..75f5a934 --- /dev/null +++ b/internal/cmd/catalog.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "github.com/openshift-pipelines/tektoncd-catalog/internal/config" + "github.com/openshift-pipelines/tektoncd-catalog/internal/runner" + "github.com/spf13/cobra" +) + +func CatalogCmd(cfg *config.Config) *cobra.Command { + catalogCmd := &cobra.Command{ + Use: "catalog", + Long: `Catalog management commands.`, + } + + catalogCmd.AddCommand(runner.NewRunner(cfg, NewCatalogGenerateCmd()).Cmd()) + catalogCmd.AddCommand(runner.NewRunner(cfg, NewCatalogGenerateFromExternalCmd()).Cmd()) + catalogCmd.AddCommand(runner.NewRunner(cfg, NewCatalogExternalsCmd()).Cmd()) + + return catalogCmd +} diff --git a/internal/cmd/externals.go b/internal/cmd/externals.go new file mode 100644 index 00000000..dd0bff33 --- /dev/null +++ b/internal/cmd/externals.go @@ -0,0 +1,125 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "os" + "path" + "strings" + + "github.com/openshift-pipelines/tektoncd-catalog/internal/config" + fc "github.com/openshift-pipelines/tektoncd-catalog/internal/fetcher/config" + "github.com/openshift-pipelines/tektoncd-catalog/internal/runner" + "github.com/spf13/cobra" +) + +// ExternalsCmd represents the "externals" subcommand to externals the signature of a resource file. +type ExternalsCmd struct { + cmd *cobra.Command // cobra command definition + config string // path for the catalog configuration file +} + +var _ runner.SubCommand = &ExternalsCmd{} + +const externalsLongDescription = `# catalog-cd externals + +TODO +` + +// Cmd exposes the cobra command instance. +func (v *ExternalsCmd) Cmd() *cobra.Command { + return v.cmd +} + +// Complete asserts the required flags are informed, and the last argument is the resource file for +// signature verification. +func (v *ExternalsCmd) Complete(_ *config.Config, args []string) error { + if v.config == "" { + return fmt.Errorf("flag --config is required") + } + + if len(args) != 0 { + return fmt.Errorf("externals takes no argument") + } + return nil +} + +// Validate asserts all the required files exists. +func (v *ExternalsCmd) Validate() error { + required := []string{ + v.config, + } + for _, f := range required { + if _, err := os.Stat(f); err != nil { + return err + } + } + return nil +} + +type GitHubRunObject struct { + Name string `json:"name"` + URL string `json:"url"` + Type string `json:"type"` + IgnoreVersions string `json:"ignoreVersions"` +} + +type GitHubMatrixObject struct { + Include []GitHubRunObject `json:"include"` +} + +// Run +func (v *ExternalsCmd) Run(cfg *config.Config) error { + e, err := fc.LoadExternal(v.config) + if err != nil { + return err + } + m := GitHubMatrixObject{} + for _, repository := range e.Repositories { + types := repository.Types + if len(types) == 0 { + types = []string{"tasks", "pipelines"} + } + ignoreVersions := "" + if len(repository.IgnoreVersions) > 0 { + ignoreVersions = strings.Join(repository.IgnoreVersions, ",") + } + for _, t := range types { + name := repository.Name + if name == "" { + name = path.Base(repository.URL) + } + o := GitHubRunObject{ + Name: name, + URL: repository.URL, + Type: t, + IgnoreVersions: ignoreVersions, + } + m.Include = append(m.Include, o) + } + } + j, err := json.Marshal(m) + if err != nil { + return err + } + fmt.Fprintf(cfg.Stream.Out, "%s\n", j) + return nil +} + +// NewCatalogExternalsCmd instantiates the "externals" subcommand. +func NewCatalogExternalsCmd() runner.SubCommand { + v := &ExternalsCmd{ + cmd: &cobra.Command{ + Use: "externals", + Args: cobra.ExactArgs(0), + Long: externalsLongDescription, + Short: "Verifies the resource file signature", + SilenceUsage: true, + }, + } + + f := v.cmd.PersistentFlags() + f.StringVar(&v.config, "config", v.config, "path of the catalog configuration file") + + return v +} diff --git a/internal/cmd/generate-from.go b/internal/cmd/generate-from.go new file mode 100644 index 00000000..d9ec2fed --- /dev/null +++ b/internal/cmd/generate-from.go @@ -0,0 +1,116 @@ +package cmd + +import ( + "fmt" + "path" + "strings" + + "github.com/cli/go-gh/v2/pkg/api" + "github.com/openshift-pipelines/tektoncd-catalog/internal/catalog" + "github.com/openshift-pipelines/tektoncd-catalog/internal/config" + fc "github.com/openshift-pipelines/tektoncd-catalog/internal/fetcher/config" + "github.com/openshift-pipelines/tektoncd-catalog/internal/runner" + "github.com/spf13/cobra" +) + +// GenerateFromExternalCmd represents the "generate" subcommand to generate the signature of a resource file. +type GenerateFromExternalCmd struct { + cmd *cobra.Command // cobra command definition + name string // name of the repository to pull (a bit useless) + url string // url of the repository to pull + resourceType string // type of resource to pull + ignoreVersions string // versions to ignore while pulling + target string // path to the folder where we want to generate the catalog +} + +var _ runner.SubCommand = &GenerateFromExternalCmd{} + +const generateLongFromExternalDescription = `# catalog-cd generate + +Generates a file-based catalog in the target folder, based of a configuration file. + + $ catalog-cd generate \ + --config="/path/to/external.yaml" \ + /path/to/catalog/target +` + +// Cmd exposes the cobra command instance. +func (v *GenerateFromExternalCmd) Cmd() *cobra.Command { + return v.cmd +} + +// Complete asserts the required flags are informed, and the last argument is the resource file for +// signature verification. +func (v *GenerateFromExternalCmd) Complete(_ *config.Config, args []string) error { + if v.url == "" { + return fmt.Errorf("flag --config is required") + } + if v.resourceType == "" { + return fmt.Errorf("flag --resourceType is required") + } + + if len(args) != 1 { + return fmt.Errorf("you must specify a target to generate the catalog in.") + } + v.target = args[0] + return nil +} + +// Validate asserts all the required files exists. +func (v *GenerateFromExternalCmd) Validate() error { + return nil +} + +// Run wrapper around "cosign generate-blob" command. +func (v *GenerateFromExternalCmd) Run(cfg *config.Config) error { + cfg.Infof("Generating a partial catalog from %s (type: %s)\n", v.url, v.resourceType) + ghclient, err := api.DefaultRESTClient() + if err != nil { + return err + } + + name := v.name + if name == "" { + name = path.Base(v.url) + } + ignoreVersions := []string{} + if v.ignoreVersions != "" { + ignoreVersions = strings.Split(v.ignoreVersions, ",") + } + + e := fc.External{ + Repositories: []fc.Repository{{ + Name: name, + URL: v.url, + Types: []string{v.resourceType}, + IgnoreVersions: ignoreVersions, + }}, + } + c, err := catalog.FetchFromExternals(e, ghclient) + if err != nil { + return err + } + + return catalog.GenerateFilesystem(v.target, c) +} + +// NewCatalogGenerateFromExternalCmd instantiates the "generate" subcommand. +func NewCatalogGenerateFromExternalCmd() runner.SubCommand { + v := &GenerateFromExternalCmd{ + cmd: &cobra.Command{ + Use: "generate-from", + Args: cobra.ExactArgs(1), + Long: generateLongFromExternalDescription, + Short: "Verifies the resource file signature", + SilenceUsage: true, + }, + } + + f := v.cmd.PersistentFlags() + f.StringVar(&v.name, "name", "", "name of the repository to pull") + f.StringVar(&v.url, "url", "", "url of the repository to pull") + f.StringVar(&v.resourceType, "type", "", "type of resource to pull") + f.StringVar(&v.ignoreVersions, "ignore-versions", "", "versions to ignore while pulling") + + return v +} diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index 29411267..9a21301d 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -43,7 +43,7 @@ func (v *GenerateCmd) Complete(_ *config.Config, args []string) error { } if len(args) != 1 { - return fmt.Errorf("you must specify a target to generate the catalog in") + return fmt.Errorf("you must specify a target to generate the catalog in.") } v.target = args[0] return nil @@ -83,11 +83,11 @@ func (v *GenerateCmd) Run(cfg *config.Config) error { return catalog.GenerateFilesystem(v.target, c) } -// NewGenerateCatalogCmd instantiates the "generate" subcommand. -func NewGenerateCatalogCmd() runner.SubCommand { +// NewCatalogGenerateCmd instantiates the "generate" subcommand. +func NewCatalogGenerateCmd() runner.SubCommand { v := &GenerateCmd{ cmd: &cobra.Command{ - Use: "generate-catalog", + Use: "generate", Args: cobra.ExactArgs(1), Long: generateLongDescription, Short: "Verifies the resource file signature", diff --git a/internal/cmd/root.go b/internal/cmd/root.go index ccf6ce47..3d1992b6 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -20,9 +20,10 @@ func NewRootCmd(stream *tkncli.Stream) *cobra.Command { rootCmd.AddCommand(runner.NewRunner(cfg, NewProbeCmd()).Cmd()) rootCmd.AddCommand(runner.NewRunner(cfg, NewRenderCmd()).Cmd()) rootCmd.AddCommand(runner.NewRunner(cfg, NewVerifyCmd()).Cmd()) - rootCmd.AddCommand(runner.NewRunner(cfg, NewGenerateCatalogCmd()).Cmd()) rootCmd.AddCommand(runner.NewRunner(cfg, NewReleaseCmd()).Cmd()) rootCmd.AddCommand(runner.NewRunner(cfg, NewSignCmd()).Cmd()) + rootCmd.AddCommand(CatalogCmd(cfg)) + return rootCmd }