Merge pull request #56 from vdemeester/catalog-cd-generate-update

Some `catalog-cd` release and generate-catalog adjustement

.github/workflows/generate-catalogs.yaml

@@ -19,7 +19,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          go run ./cmd/catalog-cd generate --config ./experimental/externals.yaml experimental
+          go run ./cmd/catalog-cd generate-catalog --config ./experimental/externals.yaml experimental
       - uses: actions/upload-artifact@v3
         with:
           name: experimental-catalog-artifact
@@ -40,7 +40,7 @@ jobs:
           mkdir -p stable/tasks stable/pipelines
           cp -fR tasks/* stable/tasks
           cp -fR pipelines/* stable/pipelines
-          go run ./cmd/catalog-cd generate --config ./externals.yaml stable
+          go run ./cmd/catalog-cd generate-catalog --config ./externals.yaml stable
       - uses: actions/upload-artifact@v3
         with:
           name: stable-catalog-artifact

internal/catalog/catalog.go

@@ -1,38 +1,27 @@
 package catalog
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/cli/go-gh/v2/pkg/api"
 	"github.com/openshift-pipelines/tektoncd-catalog/internal/fetcher"
 	"github.com/openshift-pipelines/tektoncd-catalog/internal/fetcher/config"
 )
 
-func FetchFromExternal(e config.External, client *api.RESTClient) (Catalog, error) {
+func FetchFromExternals(e config.External, client *api.RESTClient) (Catalog, error) {
 	c := Catalog{
-		Tasks:     map[string]Task{},
-		Pipelines: map[string]Pipeline{},
+		Resources: map[string]Resource{},
 	}
 	for _, r := range e.Repositories {
-		var fetchTask, fetchPipeline bool
-		if r.Types == nil {
-			fetchTask = true
-			fetchPipeline = true
-		} else {
-			for _, t := range r.Types {
-				if t == "tasks" {
-					fetchTask = true
-				}
-				if t == "pipelines" {
-					fetchPipeline = true
-				}
-			}
-		}
+		fmt.Fprintln(os.Stderr, "Fetching", r.Name, "("+r.URL+")")
+		c.Resources[r.Name] = Resource{}
+
 		m, err := fetcher.FetchContractsFromRepository(r, client)
 		if err != nil {
 			return c, err
@@ -44,144 +33,102 @@ func FetchFromExternal(e config.External, client *api.RESTClient) (Catalog, erro
 			}
 		}
 
-		for version, contract := range m {
-			if fetchTask {
-				if contract.Catalog.Resources != nil {
-					for _, task := range contract.Catalog.Resources.Tasks {
-						if _, ok := c.Tasks[task.Name]; !ok {
-							// task doesn't exists yet, creating it
-							c.Tasks[task.Name] = Task{
-								Versions: map[string]VersionnedTask{},
-							}
-						}
-						if _, ok := c.Tasks[task.Name].Versions[version]; ok {
-							//  name/version confict
-							return c, fmt.Errorf("Task %s has a version conflict (%s)", task.Name, r.URL)
-						}
-						downloadURL := task.Filename
-						if !strings.HasPrefix(task.Filename, "https://") {
-							downloadURL = fmt.Sprintf("%s/releases/download/%s/%s", r.URL, version, task.Filename)
-						}
-						c.Tasks[task.Name].Versions[version] = VersionnedTask{
-							DownloadURL: downloadURL,
-							// Bundle:      task.Bundle, // FIXME: add bundle support
-						}
-					}
-				}
-			}
-			if fetchPipeline {
-				if contract.Catalog.Resources != nil {
-					for _, pipeline := range contract.Catalog.Resources.Pipelines {
-						if _, ok := c.Pipelines[pipeline.Name]; !ok {
-							// pipeline doesn't exists yet, creating it
-							c.Pipelines[pipeline.Name] = Pipeline{
-								Versions: map[string]VersionnedPipeline{},
-							}
-						}
-						if _, ok := c.Pipelines[pipeline.Name].Versions[version]; ok {
-							// name/version confict
-							return c, fmt.Errorf("Pipeline %s has a version conflict (%s)", pipeline.Name, r.URL)
-						}
-						downloadURL := pipeline.Filename
-						if !strings.HasPrefix(pipeline.Filename, "https://") {
-							downloadURL = fmt.Sprintf("%s/releases/download/%s/%s", r.URL, version, pipeline.Filename)
-						}
-						c.Pipelines[pipeline.Name].Versions[version] = VersionnedPipeline{
-							DownloadURL: downloadURL,
-							// Bundle:      pipeline.Bundle, // FIXME: add bundle support
-						}
-					}
-				}
-			}
+		for version, _ := range m {
+			resourcesDownloaldURI := fmt.Sprintf("%s/releases/download/%s/%s", r.URL, version, "resources.tar.gz")
+			c.Resources[r.Name][version] = resourcesDownloaldURI
 		}
 	}
 	return c, nil
 }
 
 func GenerateFilesystem(path string, c Catalog) error {
-	if err := generateTasksFilesystem(filepath.Join(path, "tasks"), c.Tasks); err != nil {
-		return fmt.Errorf("Failed to create the tasks filesystem: %w", err)
-	}
-	if err := generatePipelinesFilesystem(filepath.Join(path, "pipelines"), c.Pipelines); err != nil {
-		return fmt.Errorf("Failed to create the tasks filesystem: %w", err)
-	}
-	return nil
-}
-
-func generateTasksFilesystem(path string, tasks map[string]Task) error {
-	for name, t := range tasks {
-		for version, task := range t.Versions {
-			taskfolder := filepath.Join(path, name, version)
-			if err := os.MkdirAll(taskfolder, os.ModePerm); err != nil {
-				return err
-			}
-			taskfile := filepath.Join(taskfolder, fmt.Sprintf("%s.yaml", name))
-			if err := fetchAndWrite(taskfile, task.DownloadURL); err != nil {
-				return fmt.Errorf("Couldn't fetch %s in %s: %w", task.DownloadURL, taskfile, err)
-			}
-		}
-	}
-	return nil
-}
-
-func generatePipelinesFilesystem(path string, pipelines map[string]Pipeline) error {
-	for name, t := range pipelines {
-		for version, pipeline := range t.Versions {
-			pipelinefolder := filepath.Join(path, name, version)
-			if err := os.MkdirAll(filepath.Join(path, name, version), os.ModePerm); err != nil {
-				return err
-			}
-			pipelinefile := filepath.Join(pipelinefolder, fmt.Sprintf("%s.yaml", name))
-			if err := fetchAndWrite(pipelinefile, pipeline.DownloadURL); err != nil {
-				return fmt.Errorf("Couldn't fetch %s in %s: %w", pipeline.DownloadURL, pipelinefile, err)
+	for name, resource := range c.Resources {
+		fmt.Fprintf(os.Stderr, "# Fetching resource %s\n", name)
+		for version, uri := range resource {
+			fmt.Fprintf(os.Stderr, "## Fetching version %s\n", version)
+			if err := fetchAndExtract(path, uri, version); err != nil {
+				fmt.Fprintf(os.Stderr, "Failed to fetch resource %s: %v, skipping\n", uri, err)
+				continue
 			}
 		}
 	}
 	return nil
 }
 
-func fetchAndWrite(file, url string) error {
+func fetchAndExtract(path, url, version string) error {
 	resp, err := http.Get(url)
 	if err != nil {
 		return err
 	}
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("Status error: %v", resp.StatusCode)
 	}
-	w, err := os.Create(file)
-	if err != nil {
-		return err
-	}
-	defer w.Close()
+	return untar(path, version, resp.Body)
+}
 
-	_, err = io.Copy(w, resp.Body)
+func untar(dst, version string, r io.Reader) error {
+	gzr, err := gzip.NewReader(r)
 	if err != nil {
 		return err
 	}
-	return nil
-}
+	defer gzr.Close()
 
-// Catalog is a struct that represent a "file-based" catalog
-// file-based catalog
-type Catalog struct {
-	Tasks     map[string]Task
-	Pipelines map[string]Pipeline
-}
+	tr := tar.NewReader(gzr)
 
-type Task struct {
-	Versions map[string]VersionnedTask
-}
+	for {
+		header, err := tr.Next()
+		switch {
+		// if no more files are found return
+		case err == io.EOF:
+			return nil
+		// return any other error
+		case err != nil:
+			return err
+		// if the header is nil, just skip it (not sure how this happens)
+		case header == nil:
+			continue
+		}
 
-type VersionnedTask struct {
-	DownloadURL string
-	Bundle      string
-}
+		// the target location where the dir/file should be created
+		filename := filepath.Base(header.Name)
+		targetFolder := filepath.Join(dst, filepath.Dir(header.Name), version)
+		target := filepath.Join(targetFolder, filename)
 
-type Pipeline struct {
-	Versions map[string]VersionnedPipeline
+		if err := os.MkdirAll(targetFolder, os.ModePerm); err != nil {
+			return err
+		}
+		// the following switch could also be done using fi.Mode(), not sure if there
+		// a benefit of using one vs. the other.
+		// fi := header.FileInfo()
+
+		// check the file type
+		switch header.Typeflag {
+		// if its a dir and it doesn't exist create it
+		case tar.TypeDir:
+			if _, err := os.Stat(target); err != nil {
+				if err := os.MkdirAll(target, 0755); err != nil {
+					return err
+				}
+			}
+		// if it's a file create it
+		case tar.TypeReg:
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
+			if err != nil {
+				return err
+			}
+			// copy over contents
+			if _, err := io.Copy(f, tr); err != nil {
+				return err
+			}
+			// manually close here after each file operation; defering would cause each file close
+			// to wait until all operations have completed.
+			f.Close()
+		}
+	}
 }
 
-type VersionnedPipeline struct {
-	DownloadURL string
-	Bundle      string
+type Catalog struct {
+	Resources map[string]Resource
 }
+
+type Resource map[string]string

internal/catalog/catalog_test.go

@@ -43,7 +43,7 @@ func TestFetchFromExternal(t *testing.T) {
 			Types: []string{"tasks"},
 		}},
 	}
-	c, err := catalog.FetchFromExternal(e, client)
+	c, err := catalog.FetchFromExternals(e, client)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/cmd/generate.go

@@ -75,19 +75,19 @@ func (v *GenerateCmd) Run(cfg *config.Config) error {
 	if err != nil {
 		return err
 	}
-	c, err := catalog.FetchFromExternal(e, ghclient)
+	c, err := catalog.FetchFromExternals(e, ghclient)
 	if err != nil {
 		return err
 	}
 
 	return catalog.GenerateFilesystem(v.target, c)
 }
 
-// NewGenerateCmd instantiates the "generate" subcommand.
-func NewGenerateCmd() runner.SubCommand {
+// NewGenerateCatalogCmd instantiates the "generate" subcommand.
+func NewGenerateCatalogCmd() runner.SubCommand {
 	v := &GenerateCmd{
 		cmd: &cobra.Command{
-			Use:          "generate",
+			Use:          "generate-catalog",
 			Args:         cobra.ExactArgs(1),
 			Long:         generateLongDescription,
 			Short:        "Verifies the resource file signature",