fix: catalog import non-terraform modules

pkg/bus/setup.go

@@ -59,7 +59,8 @@ func Setup(ctx context.Context, opts SetupOptions) (err error) {
 	}
 
 	// Catalog.
-	err = catalog.AddSubscriber("sync-catalog", pkgcatalog.Sync)
+	err = catalog.AddSubscriber("sync-catalog",
+		pkgcatalog.CatalogSync(opts.ModelClient).Do)
 	if err != nil {
 		return
 	}

pkg/catalog/catalog.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/drone/go-scm/scm"
 
-	catalogbus "github.com/seal-io/seal/pkg/bus/catalog"
+	"github.com/seal-io/seal/pkg/bus/catalog"
 	"github.com/seal-io/seal/pkg/dao/model"
 	"github.com/seal-io/seal/pkg/dao/model/template"
 	"github.com/seal-io/seal/pkg/dao/types"
@@ -79,50 +79,6 @@ func GetRepos(ctx context.Context, c *model.Catalog) ([]*scm.Repository, error)
 	return repos, nil
 }
 
-// Sync the given catalog, it will create or update templates from the given catalog.
-func Sync(ctx context.Context, busMessage catalogbus.BusMessage) (err error) {
-	var (
-		logger = log.WithName("catalog")
-
-		mc = busMessage.TransactionalModelClient
-		c  = busMessage.Refer
-	)
-
-	gopool.Go(func() {
-		subCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-		defer cancel()
-
-		err := SyncTemplates(subCtx, mc, c)
-		if err != nil {
-			status.CatalogStatusInitialized.False(c, err.Error())
-			logger.Errorf("failed to sync catalog %s templates: %v", c.Name, err)
-		} else {
-			status.CatalogStatusReady.Reset(c, "")
-			status.CatalogStatusReady.True(c, "")
-		}
-
-		c.Status.SetSummary(status.WalkCatalog(&c.Status))
-		update := mc.Catalogs().UpdateOne(c).
-			SetStatus(c.Status)
-
-		syncResult, err := getSyncResult(subCtx, mc, c)
-		if err != nil {
-			logger.Errorf("failed to update sync info: %v", err)
-		}
-
-		if syncResult != nil {
-			update.SetSync(syncResult)
-		}
-
-		rerr := update.Exec(subCtx)
-		if rerr != nil {
-			logger.Errorf("failed to update catalog %s status: %v", c.Name, rerr)
-		}
-	})
-
-	return nil
-}
-
 func getSyncResult(ctx context.Context, mc model.ClientSet, c *model.Catalog) (*types.CatalogSync, error) {
 	var (
 		catalogSync = &types.CatalogSync{
@@ -217,3 +173,53 @@ func GetOrgFromGitURL(str string) (string, error) {
 
 	return "", fmt.Errorf("invalid git url")
 }
+
+type catalogSyncer struct {
+	mc model.ClientSet
+}
+
+func CatalogSync(mc model.ClientSet) catalogSyncer {
+	return catalogSyncer{mc: mc}
+}
+
+// Do Sync the given catalog, it will create or update templates from the given catalog.
+func (cs catalogSyncer) Do(_ context.Context, busMessage catalog.BusMessage) error {
+	var (
+		logger = log.WithName("catalog")
+
+		c = busMessage.Refer
+	)
+
+	gopool.Go(func() {
+		subCtx := context.Background()
+
+		err := SyncTemplates(subCtx, cs.mc, c)
+		if err != nil {
+			status.CatalogStatusInitialized.False(c, err.Error())
+			logger.Errorf("failed to sync catalog %s templates: %v", c.Name, err)
+		} else {
+			status.CatalogStatusReady.Reset(c, "")
+			status.CatalogStatusReady.True(c, "")
+		}
+
+		c.Status.SetSummary(status.WalkCatalog(&c.Status))
+		update := cs.mc.Catalogs().UpdateOne(c).
+			SetStatus(c.Status)
+
+		syncResult, err := getSyncResult(subCtx, cs.mc, c)
+		if err != nil {
+			logger.Errorf("failed to update sync info: %v", err)
+		}
+
+		if syncResult != nil {
+			update.SetSync(syncResult)
+		}
+
+		rerr := update.Exec(subCtx)
+		if rerr != nil {
+			logger.Errorf("failed to update catalog %s status: %v", c.Name, rerr)
+		}
+	})
+
+	return nil
+}

pkg/templates/git.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/seal-io/seal/pkg/dao/model"
 	"github.com/seal-io/seal/pkg/dao/model/templateversion"
+	"github.com/seal-io/seal/pkg/dao/types"
 	"github.com/seal-io/seal/pkg/dao/types/status"
 	"github.com/seal-io/seal/pkg/vcs"
 	"github.com/seal-io/seal/pkg/vcs/driver/gitlab"
@@ -28,19 +29,14 @@ func CreateTemplateVersionsFromRepo(
 	ctx context.Context,
 	mc model.ClientSet,
 	entity *model.Template,
-	r *git.Repository,
 	versions []*version.Version,
+	versionSchema map[*version.Version]*types.TemplateSchema,
 ) error {
 	logger := log.WithName("template")
 
-	w, err := r.Worktree()
-	if err != nil {
-		return err
-	}
-
 	var ovs []string
 	// Old versions.
-	err = mc.TemplateVersions().Query().
+	err := mc.TemplateVersions().Query().
 		Select(templateversion.FieldVersion).
 		Where(templateversion.TemplateID(entity.ID)).
 		Modify(func(s *sql.Selector) {
@@ -67,7 +63,7 @@ func CreateTemplateVersionsFromRepo(
 	}
 
 	// Create template versions.
-	templateVersions, err := GetTemplateVersions(entity, w, newVersions)
+	templateVersions, err := GetTemplateVersions(entity, newVersions, versionSchema)
 	if err != nil {
 		return err
 	}
@@ -246,6 +242,11 @@ func SyncTemplateFromGitRepo(
 		return err
 	}
 
+	versions, versionSchema, err := getValidVersions(r, versions)
+	if err != nil {
+		return err
+	}
+
 	if len(versions) == 0 {
 		logger.Warnf("no versions found for %s", repo.Name)
 		return nil
@@ -285,19 +286,19 @@ func SyncTemplateFromGitRepo(
 		}
 	}()
 
-	return CreateTemplateVersionsFromRepo(ctx, mc, entity, r, versions)
+	return CreateTemplateVersionsFromRepo(ctx, mc, entity, versions, versionSchema)
 }
 
 // GetTemplateVersions retrieves template versions from a git repository.
 // It will save images to the database if they are found in the repository.
 func GetTemplateVersions(
 	entity *model.Template,
-	w *git.Worktree,
-	versions []*version.Version,
+	newVersions []*version.Version,
+	versionSchema map[*version.Version]*types.TemplateSchema,
 ) (model.TemplateVersions, error) {
 	var (
 		logger = log.WithName("catalog")
-		tvs    = make(model.TemplateVersions, 0, len(versions))
+		tvs    = make(model.TemplateVersions, 0, len(versionSchema))
 	)
 
 	u, err := transport.NewEndpoint(entity.Source)
@@ -307,29 +308,18 @@ func GetTemplateVersions(
 
 	source := u.Host + u.Path
 
-	for i := range versions {
-		v := versions[i]
+	for i := range newVersions {
+		v := newVersions[i]
 		tag := v.Original()
 
-		err := w.Reset(&git.ResetOptions{
-			Commit: plumbing.NewHash(tag),
-			Mode:   git.HardReset,
-		})
-		if err != nil {
-			logger.Warnf("failed to reset to tag %s: %v", tag, err)
-			continue
-		}
-
-		dir := w.Filesystem.Root()
-
-		schema, err := loadTerraformTemplateSchema(dir)
-		if err != nil {
-			logger.Warnf("failed to load terraform template schema: %v", err)
+		schema, ok := versionSchema[v]
+		if !ok {
+			logger.Warnf("version schema not found, version: %s", tag)
 			continue
 		}
 
-		if schema == nil {
-			logger.Warnf("terraform template schema is nil")
+		if !isValidSchema(schema) {
+			logger.Warnf("schema is invalid template: %s, version: %s", entity.Name, tag)
 			continue
 		}
 
@@ -408,3 +398,50 @@ func GetRepoFileRaw(repo *vcs.Repository, file string, c *model.Catalog) (string
 
 	return "", nil
 }
+
+// getValidVersions get valid terraform module versions.
+func getValidVersions(
+	r *git.Repository,
+	versions []*version.Version,
+) ([]*version.Version, map[*version.Version]*types.TemplateSchema, error) {
+	logger := log.WithName("template")
+
+	w, err := r.Worktree()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	validVersions := make([]*version.Version, 0, len(versions))
+	versionSchema := make(map[*version.Version]*types.TemplateSchema, 0)
+
+	for i := range versions {
+		v := versions[i]
+		tag := v.Original()
+
+		err := w.Reset(&git.ResetOptions{
+			Commit: plumbing.NewHash(tag),
+			Mode:   git.HardReset,
+		})
+		if err != nil {
+			logger.Warnf("failed to reset to tag %s: %v", tag, err)
+			continue
+		}
+
+		dir := w.Filesystem.Root()
+
+		schema, err := loadTerraformTemplateSchema(dir)
+		if err != nil {
+			logger.Warnf("failed to load terraform template schema: %v", err)
+			continue
+		}
+
+		if !isValidSchema(schema) {
+			continue
+		}
+
+		validVersions = append(validVersions, v)
+		versionSchema[v] = schema
+	}
+
+	return validVersions, versionSchema, nil
+}

pkg/templates/terraform.go

@@ -585,3 +585,7 @@ func getOutputValueFromFile(file *hcl.File) map[string][]byte {
 
 	return outputs
 }
+
+func isValidSchema(ts *types.TemplateSchema) bool {
+	return !(len(ts.Outputs) == 0 && len(ts.RequiredProviders) == 0 && len(ts.Variables) == 0)
+}

pkg/vcs/repo.go

@@ -145,7 +145,7 @@ func CloneGitRepo(link, dir string) (*git.Repository, error) {
 
 	switch endpoint.Protocol {
 	case "http", "https":
-		src = endpoint.Host + endpoint.Path
+		src = strings.TrimLeft(endpoint.String(), endpoint.Protocol+"://")
 	default:
 		src = link
 	}