diff --git a/pkg/bus/setup.go b/pkg/bus/setup.go index df809fd31..f4bbda773 100644 --- a/pkg/bus/setup.go +++ b/pkg/bus/setup.go @@ -59,7 +59,8 @@ func Setup(ctx context.Context, opts SetupOptions) (err error) { } // Catalog. - err = catalog.AddSubscriber("sync-catalog", pkgcatalog.Sync) + err = catalog.AddSubscriber("sync-catalog", + pkgcatalog.CatalogSync(opts.ModelClient).Do) if err != nil { return } diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go index 66c60e252..663e91eaa 100644 --- a/pkg/catalog/catalog.go +++ b/pkg/catalog/catalog.go @@ -9,7 +9,7 @@ import ( "github.com/drone/go-scm/scm" - catalogbus "github.com/seal-io/seal/pkg/bus/catalog" + "github.com/seal-io/seal/pkg/bus/catalog" "github.com/seal-io/seal/pkg/dao/model" "github.com/seal-io/seal/pkg/dao/model/template" "github.com/seal-io/seal/pkg/dao/types" @@ -79,50 +79,6 @@ func GetRepos(ctx context.Context, c *model.Catalog) ([]*scm.Repository, error) return repos, nil } -// Sync the given catalog, it will create or update templates from the given catalog. -func Sync(ctx context.Context, busMessage catalogbus.BusMessage) (err error) { - var ( - logger = log.WithName("catalog") - - mc = busMessage.TransactionalModelClient - c = busMessage.Refer - ) - - gopool.Go(func() { - subCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - - err := SyncTemplates(subCtx, mc, c) - if err != nil { - status.CatalogStatusInitialized.False(c, err.Error()) - logger.Errorf("failed to sync catalog %s templates: %v", c.Name, err) - } else { - status.CatalogStatusReady.Reset(c, "") - status.CatalogStatusReady.True(c, "") - } - - c.Status.SetSummary(status.WalkCatalog(&c.Status)) - update := mc.Catalogs().UpdateOne(c). - SetStatus(c.Status) - - syncResult, err := getSyncResult(subCtx, mc, c) - if err != nil { - logger.Errorf("failed to update sync info: %v", err) - } - - if syncResult != nil { - update.SetSync(syncResult) - } - - rerr := update.Exec(subCtx) - if rerr != nil { - logger.Errorf("failed to update catalog %s status: %v", c.Name, rerr) - } - }) - - return nil -} - func getSyncResult(ctx context.Context, mc model.ClientSet, c *model.Catalog) (*types.CatalogSync, error) { var ( catalogSync = &types.CatalogSync{ @@ -217,3 +173,53 @@ func GetOrgFromGitURL(str string) (string, error) { return "", fmt.Errorf("invalid git url") } + +type catalogSyncer struct { + mc model.ClientSet +} + +func CatalogSync(mc model.ClientSet) catalogSyncer { + return catalogSyncer{mc: mc} +} + +// Do Sync the given catalog, it will create or update templates from the given catalog. +func (cs catalogSyncer) Do(_ context.Context, busMessage catalog.BusMessage) error { + var ( + logger = log.WithName("catalog") + + c = busMessage.Refer + ) + + gopool.Go(func() { + subCtx := context.Background() + + err := SyncTemplates(subCtx, cs.mc, c) + if err != nil { + status.CatalogStatusInitialized.False(c, err.Error()) + logger.Errorf("failed to sync catalog %s templates: %v", c.Name, err) + } else { + status.CatalogStatusReady.Reset(c, "") + status.CatalogStatusReady.True(c, "") + } + + c.Status.SetSummary(status.WalkCatalog(&c.Status)) + update := cs.mc.Catalogs().UpdateOne(c). + SetStatus(c.Status) + + syncResult, err := getSyncResult(subCtx, cs.mc, c) + if err != nil { + logger.Errorf("failed to update sync info: %v", err) + } + + if syncResult != nil { + update.SetSync(syncResult) + } + + rerr := update.Exec(subCtx) + if rerr != nil { + logger.Errorf("failed to update catalog %s status: %v", c.Name, rerr) + } + }) + + return nil +} diff --git a/pkg/templates/git.go b/pkg/templates/git.go index 1df4d8d97..64d4fbd1c 100644 --- a/pkg/templates/git.go +++ b/pkg/templates/git.go @@ -15,6 +15,7 @@ import ( "github.com/seal-io/seal/pkg/dao/model" "github.com/seal-io/seal/pkg/dao/model/templateversion" + "github.com/seal-io/seal/pkg/dao/types" "github.com/seal-io/seal/pkg/dao/types/status" "github.com/seal-io/seal/pkg/vcs" "github.com/seal-io/seal/pkg/vcs/driver/gitlab" @@ -28,19 +29,14 @@ func CreateTemplateVersionsFromRepo( ctx context.Context, mc model.ClientSet, entity *model.Template, - r *git.Repository, versions []*version.Version, + versionSchema map[*version.Version]*types.TemplateSchema, ) error { logger := log.WithName("template") - w, err := r.Worktree() - if err != nil { - return err - } - var ovs []string // Old versions. - err = mc.TemplateVersions().Query(). + err := mc.TemplateVersions().Query(). Select(templateversion.FieldVersion). Where(templateversion.TemplateID(entity.ID)). Modify(func(s *sql.Selector) { @@ -67,7 +63,7 @@ func CreateTemplateVersionsFromRepo( } // Create template versions. - templateVersions, err := GetTemplateVersions(entity, w, newVersions) + templateVersions, err := GetTemplateVersions(entity, newVersions, versionSchema) if err != nil { return err } @@ -246,6 +242,11 @@ func SyncTemplateFromGitRepo( return err } + versions, versionSchema, err := getValidVersions(r, versions) + if err != nil { + return err + } + if len(versions) == 0 { logger.Warnf("no versions found for %s", repo.Name) return nil @@ -285,19 +286,19 @@ func SyncTemplateFromGitRepo( } }() - return CreateTemplateVersionsFromRepo(ctx, mc, entity, r, versions) + return CreateTemplateVersionsFromRepo(ctx, mc, entity, versions, versionSchema) } // GetTemplateVersions retrieves template versions from a git repository. // It will save images to the database if they are found in the repository. func GetTemplateVersions( entity *model.Template, - w *git.Worktree, - versions []*version.Version, + newVersions []*version.Version, + versionSchema map[*version.Version]*types.TemplateSchema, ) (model.TemplateVersions, error) { var ( logger = log.WithName("catalog") - tvs = make(model.TemplateVersions, 0, len(versions)) + tvs = make(model.TemplateVersions, 0, len(versionSchema)) ) u, err := transport.NewEndpoint(entity.Source) @@ -307,29 +308,18 @@ func GetTemplateVersions( source := u.Host + u.Path - for i := range versions { - v := versions[i] + for i := range newVersions { + v := newVersions[i] tag := v.Original() - err := w.Reset(&git.ResetOptions{ - Commit: plumbing.NewHash(tag), - Mode: git.HardReset, - }) - if err != nil { - logger.Warnf("failed to reset to tag %s: %v", tag, err) - continue - } - - dir := w.Filesystem.Root() - - schema, err := loadTerraformTemplateSchema(dir) - if err != nil { - logger.Warnf("failed to load terraform template schema: %v", err) + schema, ok := versionSchema[v] + if !ok { + logger.Warnf("version schema not found, version: %s", tag) continue } - if schema == nil { - logger.Warnf("terraform template schema is nil") + if !isValidSchema(schema) { + logger.Warnf("schema is invalid template: %s, version: %s", entity.Name, tag) continue } @@ -408,3 +398,50 @@ func GetRepoFileRaw(repo *vcs.Repository, file string, c *model.Catalog) (string return "", nil } + +// getValidVersions get valid terraform module versions. +func getValidVersions( + r *git.Repository, + versions []*version.Version, +) ([]*version.Version, map[*version.Version]*types.TemplateSchema, error) { + logger := log.WithName("template") + + w, err := r.Worktree() + if err != nil { + return nil, nil, err + } + + validVersions := make([]*version.Version, 0, len(versions)) + versionSchema := make(map[*version.Version]*types.TemplateSchema, 0) + + for i := range versions { + v := versions[i] + tag := v.Original() + + err := w.Reset(&git.ResetOptions{ + Commit: plumbing.NewHash(tag), + Mode: git.HardReset, + }) + if err != nil { + logger.Warnf("failed to reset to tag %s: %v", tag, err) + continue + } + + dir := w.Filesystem.Root() + + schema, err := loadTerraformTemplateSchema(dir) + if err != nil { + logger.Warnf("failed to load terraform template schema: %v", err) + continue + } + + if !isValidSchema(schema) { + continue + } + + validVersions = append(validVersions, v) + versionSchema[v] = schema + } + + return validVersions, versionSchema, nil +} diff --git a/pkg/templates/terraform.go b/pkg/templates/terraform.go index ac2cbe34a..b9d88b053 100644 --- a/pkg/templates/terraform.go +++ b/pkg/templates/terraform.go @@ -585,3 +585,7 @@ func getOutputValueFromFile(file *hcl.File) map[string][]byte { return outputs } + +func isValidSchema(ts *types.TemplateSchema) bool { + return !(len(ts.Outputs) == 0 && len(ts.RequiredProviders) == 0 && len(ts.Variables) == 0) +}