From f0c7ed5a99d8ed6eb313fe8a7dbb70e88816f7bb Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Mon, 23 Oct 2023 11:46:08 +0200 Subject: [PATCH] Hub resolver: add version constraints This let the user specify a version constraint to choose from. Version constraint looks like this: ```yaml version: ">= 0.5.0" ``` This will only choose the tasks that are greater than 0.5.0 Additional constraint operators are available, for example: ```yaml version: ">= 0.5.0, < 2.0.0" ``` This will only choose the tasks that are greater than 0.5.0 and less than 2.0.0 Signed-off-by: Chmouel Boudjnah --- cmd/resolvers/main.go | 14 +- cmd/resolvers/main_test.go | 40 ++++ docs/hub-resolver.md | 34 +++- pkg/resolution/resolver/hub/params.go | 8 +- pkg/resolution/resolver/hub/resolver.go | 98 ++++++++- pkg/resolution/resolver/hub/resolver_test.go | 201 ++++++++++++++++++- test/resolvers_test.go | 2 +- 7 files changed, 380 insertions(+), 17 deletions(-) create mode 100644 cmd/resolvers/main_test.go diff --git a/cmd/resolvers/main.go b/cmd/resolvers/main.go index 641c407785e..790b14e9def 100644 --- a/cmd/resolvers/main.go +++ b/cmd/resolvers/main.go @@ -33,8 +33,8 @@ import ( func main() { ctx := filteredinformerfactory.WithSelectors(signals.NewContext(), v1alpha1.ManagedByLabelKey) - tektonHubURL := buildHubURL(os.Getenv("TEKTON_HUB_API"), "", hub.TektonHubYamlEndpoint) - artifactHubURL := buildHubURL(os.Getenv("ARTIFACT_HUB_API"), hub.DefaultArtifactHubURL, hub.ArtifactHubYamlEndpoint) + tektonHubURL := buildHubURL(os.Getenv("TEKTON_HUB_API"), "") + artifactHubURL := buildHubURL(os.Getenv("ARTIFACT_HUB_API"), hub.DefaultArtifactHubURL) sharedmain.MainWithContext(ctx, "controller", framework.NewController(ctx, &git.Resolver{}), @@ -43,16 +43,12 @@ func main() { framework.NewController(ctx, &cluster.Resolver{})) } -func buildHubURL(configAPI, defaultURL, yamlEndpoint string) string { +func buildHubURL(configAPI, defaultURL string) string { var hubURL string if configAPI == "" { hubURL = defaultURL } else { - if !strings.HasSuffix(configAPI, "/") { - configAPI += "/" - } - hubURL = configAPI + yamlEndpoint + hubURL = configAPI } - - return hubURL + return strings.TrimSuffix(hubURL, "/") } diff --git a/cmd/resolvers/main_test.go b/cmd/resolvers/main_test.go new file mode 100644 index 00000000000..7848a8416b1 --- /dev/null +++ b/cmd/resolvers/main_test.go @@ -0,0 +1,40 @@ +package main + +import "testing" + +func TestBuildHubURL(t *testing.T) { + testCases := []struct { + name string + configAPI string + defaultURL string + expected string + }{ + { + name: "configAPI empty", + configAPI: "", + defaultURL: "https://tekton.dev", + expected: "https://tekton.dev", + }, + { + name: "configAPI not empty", + configAPI: "https://myhub.com", + defaultURL: "https://foo.com", + expected: "https://myhub.com", + }, + { + name: "defaultURL ends with slash", + configAPI: "", + defaultURL: "https://bar.com/", + expected: "https://bar.com", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + actual := buildHubURL(tc.configAPI, tc.defaultURL) + if actual != tc.expected { + t.Errorf("expected %s, but got %s", tc.expected, actual) + } + }) + } +} diff --git a/docs/hub-resolver.md b/docs/hub-resolver.md index b7cc7c64fe6..dcac0a64279 100644 --- a/docs/hub-resolver.md +++ b/docs/hub-resolver.md @@ -17,7 +17,7 @@ Use resolver type `hub`. | `type` | The type of Hub from where to pull the resource (Optional). Either `artifact` or `tekton` | Default: `artifact` | | `kind` | Either `task` or `pipeline` (Optional) | Default: `task` | | `name` | The name of the task or pipeline to fetch from the hub | `golang-build` | -| `version` | Version of task or pipeline to pull in from hub. Wrap the number in quotes! | `"0.5.0"` | +| `version` | Version or a Constraint (see [below](#version-constraint) of a task or a pipeline to pull in from. Wrap the number in quotes! | `"0.5.0"`, `">= 0.5.0"` | The Catalogs in the Artifact Hub follows the semVer (i.e.` ..0`) and the Catalogs in the Tekton Hub follows the simplified semVer (i.e. `.`). Both full and simplified semantic versioning will be accepted by the `version` parameter. The Hub Resolver will map the version to the format expected by the target Hub `type`. @@ -129,6 +129,38 @@ spec: # overall will not succeed without those parameters. ``` +### Version constraint + +Instead of a version you can specify a constraint to choose from. The constraint is a string as documented in the [go-version](https://github.com/hashicorp/go-version) library. + +Some examples: + +```yaml +params: + - name: name + value: git-clone + - name: version + value: ">=0.7.0" +``` + +Will only choose the git-clone task that is greater than version `0.7.0` + +```yaml +params: + - name: name + value: git-clone + - name: version + value: ">=0.7.0, < 2.0.0" +``` + +Will select the **latest** git-clone task that is greater than version `0.7.0` and +less than version `2.0.0`, so if the latest task is the version `0.9.0` it will +be selected. + +Other operators for selection are available for comparisons, see the +[go-version](https://github.com/hashicorp/go-version/blob/644291d14038339745c2d883a1a114488e30b702/constraint.go#L40C2-L48) +source code. + --- Except as otherwise noted, the content of this page is licensed under the diff --git a/pkg/resolution/resolver/hub/params.go b/pkg/resolution/resolver/hub/params.go index f9bfd711055..6c77736b48f 100644 --- a/pkg/resolution/resolver/hub/params.go +++ b/pkg/resolution/resolver/hub/params.go @@ -14,14 +14,20 @@ limitations under the License. package hub // DefaultArtifactHubURL is the default url for the Artifact hub api -const DefaultArtifactHubURL = "https://artifacthub.io/api/v1/packages/tekton-%s/%s/%s/%s" +const DefaultArtifactHubURL = "https://artifacthub.io" // TektonHubYamlEndpoint is the suffix for a private custom Tekton hub instance const TektonHubYamlEndpoint = "v1/resource/%s/%s/%s/%s/yaml" +// DefaultTektonHubListTasksEndpoint +const TektonHubListTasksEndpoint = "v1/resource/%s/%s/%s" + // ArtifactHubYamlEndpoint is the suffix for a private custom Artifact hub instance const ArtifactHubYamlEndpoint = "api/v1/packages/tekton-%s/%s/%s/%s" +// ArtifactHubListTasksEndpoint +const ArtifactHubListTasksEndpoint = "api/v1/packages/tekton-%s/%s/%s" + // ParamName is the parameter defining what the layer name in the bundle // image is. const ParamName = "name" diff --git a/pkg/resolution/resolver/hub/resolver.go b/pkg/resolution/resolver/hub/resolver.go index 2b73aeb6941..abbc19c1985 100644 --- a/pkg/resolution/resolver/hub/resolver.go +++ b/pkg/resolution/resolver/hub/resolver.go @@ -24,6 +24,7 @@ import ( "net/http" "strings" + goversion "github.com/hashicorp/go-version" resolverconfig "github.com/tektoncd/pipeline/pkg/apis/config/resolver" pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" "github.com/tektoncd/pipeline/pkg/resolution/common" @@ -121,6 +122,14 @@ func (r *Resolver) Resolve(ctx context.Context, params []pipelinev1.Param) (fram return nil, fmt.Errorf("failed to validate params: %w", err) } + if constraint, err := goversion.NewConstraint(paramsMap[ParamVersion]); err == nil { + chosen, err := r.resolveVersionConstraint(ctx, paramsMap, constraint) + if err != nil { + return nil, err + } + paramsMap[ParamVersion] = chosen.String() + } + resVer, err := resolveVersion(paramsMap[ParamVersion], paramsMap[ParamType]) if err != nil { return nil, err @@ -130,7 +139,8 @@ func (r *Resolver) Resolve(ctx context.Context, params []pipelinev1.Param) (fram // call hub API switch paramsMap[ParamType] { case ArtifactHubType: - url := fmt.Sprintf(r.ArtifactHubURL, paramsMap[ParamKind], paramsMap[ParamCatalog], paramsMap[ParamName], paramsMap[ParamVersion]) + url := fmt.Sprintf(fmt.Sprintf("%s/%s", r.ArtifactHubURL, ArtifactHubYamlEndpoint), + paramsMap[ParamKind], paramsMap[ParamCatalog], paramsMap[ParamName], paramsMap[ParamVersion]) resp := artifactHubResponse{} if err := fetchHubResource(ctx, url, &resp); err != nil { return nil, fmt.Errorf("fail to fetch Artifact Hub resource: %w", err) @@ -140,7 +150,8 @@ func (r *Resolver) Resolve(ctx context.Context, params []pipelinev1.Param) (fram Content: []byte(resp.Data.YAML), }, nil case TektonHubType: - url := fmt.Sprintf(r.TektonHubURL, paramsMap[ParamCatalog], paramsMap[ParamKind], paramsMap[ParamName], paramsMap[ParamVersion]) + url := fmt.Sprintf(fmt.Sprintf("%s/%s", r.TektonHubURL, TektonHubYamlEndpoint), + paramsMap[ParamCatalog], paramsMap[ParamKind], paramsMap[ParamName], paramsMap[ParamVersion]) resp := tektonHubResponse{} if err := fetchHubResource(ctx, url, &resp); err != nil { return nil, fmt.Errorf("fail to fetch Tekton Hub resource: %w", err) @@ -218,7 +229,6 @@ func fetchHubResource(ctx context.Context, apiEndpoint string, v interface{}) er if err != nil { return fmt.Errorf("error unmarshalling json response: %w", err) } - return nil } @@ -256,6 +266,88 @@ func resolveCatalogName(paramsMap, conf map[string]string) (string, error) { return paramsMap[ParamCatalog], nil } +type artifactHubavailableVersionsResults struct { + Version string `json:"version"` + Prerelease bool `json:"prerelease"` +} + +type artifactHubListResult struct { + AvailableVersions []artifactHubavailableVersionsResults `json:"available_versions"` + Version string `json:"version"` +} + +type tektonHubListResultVersion struct { + Version string `json:"version"` +} + +type tektonHubListDataResult struct { + Versions []tektonHubListResultVersion `json:"versions"` +} + +type tektonHubListResult struct { + Data tektonHubListDataResult `json:"data"` +} + +func (r *Resolver) resolveVersionConstraint(ctx context.Context, paramsMap map[string]string, constraint goversion.Constraints) (*goversion.Version, error) { + var ret *goversion.Version + if paramsMap[ParamType] == ArtifactHubType { + allVersionsURL := fmt.Sprintf("%s/%s", r.ArtifactHubURL, fmt.Sprintf( + ArtifactHubListTasksEndpoint, + paramsMap[ParamKind], paramsMap[ParamCatalog], paramsMap[ParamName])) + resp := artifactHubListResult{} + if err := fetchHubResource(ctx, allVersionsURL, &resp); err != nil { + return nil, fmt.Errorf("fail to fetch Artifact Hub resource: %w", err) + } + for _, vers := range resp.AvailableVersions { + if vers.Prerelease { + continue + } + checkV, err := goversion.NewVersion(vers.Version) + if err != nil { + return nil, fmt.Errorf("fail to parse version %s from %s: %w", ArtifactHubType, vers.Version, err) + } + if checkV == nil { + continue + } + if constraint.Check(checkV) { + if ret != nil && ret.GreaterThan(checkV) { + continue + } + // TODO(chmouel): log constraint result in controller + ret = checkV + } + } + } else if paramsMap[ParamType] == TektonHubType { + allVersionsURL := fmt.Sprintf("%s/%s", r.TektonHubURL, + fmt.Sprintf(TektonHubListTasksEndpoint, + paramsMap[ParamCatalog], paramsMap[ParamKind], paramsMap[ParamName])) + resp := tektonHubListResult{} + if err := fetchHubResource(ctx, allVersionsURL, &resp); err != nil { + return nil, fmt.Errorf("fail to fetch Tekton Hub resource: %w", err) + } + for _, vers := range resp.Data.Versions { + checkV, err := goversion.NewVersion(vers.Version) + if err != nil { + return nil, fmt.Errorf("fail to parse version %s from %s: %w", TektonHubType, vers, err) + } + if checkV == nil { + continue + } + if constraint.Check(checkV) { + if ret != nil && ret.GreaterThan(checkV) { + continue + } + // TODO(chmouel): log constraint result in controller + ret = checkV + } + } + } + if ret == nil { + return nil, fmt.Errorf("no version found for constraint %s", paramsMap[ParamVersion]) + } + return ret, nil +} + // the Artifact Hub follows the semVer (i.e. ..0) // the Tekton Hub follows the simplified semVer (i.e. .) // for resolution request with "artifact" type, we append ".0" suffix if the input version is simplified semVer diff --git a/pkg/resolution/resolver/hub/resolver_test.go b/pkg/resolution/resolver/hub/resolver_test.go index e91aa0be3d7..e6ba327c969 100644 --- a/pkg/resolution/resolver/hub/resolver_test.go +++ b/pkg/resolution/resolver/hub/resolver_test.go @@ -18,9 +18,12 @@ package hub import ( "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" + "path/filepath" + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -177,6 +180,200 @@ func TestValidateParamsConflictingKindName(t *testing.T) { } } +func TestResolveConstraint(t *testing.T) { + tests := []struct { + name string + wantErr bool + kind string + version string + catalog string + taskName string + hubType string + resultTask any + resultList any + expectedRes string + expectedTaskVersion string + expectedErr error + }{ + { + name: "good/tekton hub/versions constraints", + kind: "task", + version: ">= 0.1", + catalog: "Tekton", + taskName: "something", + hubType: TektonHubType, + expectedRes: "some content", + resultTask: &tektonHubResponse{ + Data: tektonHubDataResponse{ + YAML: "some content", + }, + }, + resultList: &tektonHubListResult{ + Data: tektonHubListDataResult{ + Versions: []tektonHubListResultVersion{ + { + Version: "0.1", + }, + }, + }, + }, + }, { + name: "good/tekton hub/only the greatest of the constraint", + kind: "task", + version: ">= 0.1", + catalog: "Tekton", + taskName: "something", + hubType: TektonHubType, + expectedRes: "some content", + resultTask: &tektonHubResponse{ + Data: tektonHubDataResponse{ + YAML: "some content", + }, + }, + resultList: &tektonHubListResult{ + Data: tektonHubListDataResult{ + Versions: []tektonHubListResultVersion{ + { + Version: "0.1", + }, + { + Version: "0.2", + }, + }, + }, + }, + expectedTaskVersion: "0.2", + }, { + name: "good/artifact hub/only the greatest of the constraint", + kind: "task", + version: ">= 0.1", + catalog: "Tekton", + taskName: "something", + hubType: ArtifactHubType, + expectedRes: "some content", + resultTask: &artifactHubResponse{ + Data: artifactHubDataResponse{ + YAML: "some content", + }, + }, + resultList: &artifactHubListResult{ + AvailableVersions: []artifactHubavailableVersionsResults{ + { + Version: "0.1.0", + }, + { + Version: "0.2.0", + }, + }, + }, + expectedTaskVersion: "0.2.0", + }, { + name: "good/artifact hub/versions constraints", + kind: "task", + version: ">= 0.1.0", + catalog: "Tekton", + taskName: "something", + hubType: ArtifactHubType, + expectedRes: "some content", + resultTask: &artifactHubResponse{ + Data: artifactHubDataResponse{ + YAML: "some content", + }, + }, + resultList: &artifactHubListResult{ + AvailableVersions: []artifactHubavailableVersionsResults{ + { + Version: "0.1.0", + }, + }, + }, + }, { + name: "bad/artifact hub/no matching constraints", + kind: "task", + version: ">= 0.2.0", + catalog: "Tekton", + taskName: "something", + hubType: ArtifactHubType, + resultList: &artifactHubListResult{ + AvailableVersions: []artifactHubavailableVersionsResults{ + { + Version: "0.1.0", + }, + }, + }, + expectedErr: fmt.Errorf("no version found for constraint >= 0.2.0"), + }, { + name: "bad/tekton hub/no matching constraints", + kind: "task", + version: ">= 0.2.0", + catalog: "Tekton", + taskName: "something", + hubType: ArtifactHubType, + resultList: &tektonHubListResult{ + Data: tektonHubListDataResult{ + Versions: []tektonHubListResultVersion{ + { + Version: "0.1", + }, + }, + }, + }, + expectedErr: fmt.Errorf("no version found for constraint >= 0.2.0"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var ret any + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + listURL := fmt.Sprintf(ArtifactHubListTasksEndpoint, tt.kind, tt.catalog, tt.taskName) + if tt.hubType == TektonHubType { + listURL = fmt.Sprintf(TektonHubListTasksEndpoint, tt.catalog, tt.kind, tt.taskName) + } + if r.URL.Path == "/"+listURL { + // encore result list as json + ret = tt.resultList + } else { + if tt.expectedTaskVersion != "" { + version := filepath.Base(r.URL.Path) + if tt.hubType == TektonHubType { + version = strings.Split(r.URL.Path, "/")[6] + } + if tt.expectedTaskVersion != version { + t.Fatalf("unexpected version: %s wanted: %s", version, tt.expectedTaskVersion) + } + } + ret = tt.resultTask + } + output, _ := json.Marshal(ret) + fmt.Fprintf(w, string(output)) + })) + + resolver := &Resolver{ + TektonHubURL: svr.URL, + ArtifactHubURL: svr.URL, + } + params := map[string]string{ + ParamKind: tt.kind, + ParamName: tt.taskName, + ParamVersion: tt.version, + ParamCatalog: tt.catalog, + ParamType: tt.hubType, + } + output, err := resolver.Resolve(contextWithConfig(), toParams(params)) + if tt.expectedErr != nil { + checkExpectedErr(t, tt.expectedErr, err) + } else { + if err != nil { + t.Fatalf("unexpected error resolving: %v", err) + } + if d := cmp.Diff(tt.expectedRes, string(output.Data())); d != "" { + t.Errorf("unexpected resource from Resolve: %s", diff.PrintWantGot(d)) + } + } + }) + } +} + func TestResolveVersion(t *testing.T) { testCases := []struct { name string @@ -388,8 +585,8 @@ func TestResolve(t *testing.T) { })) resolver := &Resolver{ - TektonHubURL: svr.URL + "/" + TektonHubYamlEndpoint, - ArtifactHubURL: svr.URL + "/" + ArtifactHubYamlEndpoint, + TektonHubURL: svr.URL, + ArtifactHubURL: svr.URL, } params := map[string]string{ diff --git a/test/resolvers_test.go b/test/resolvers_test.go index 1464bbdc4df..4d1449531f9 100644 --- a/test/resolvers_test.go +++ b/test/resolvers_test.go @@ -194,7 +194,7 @@ spec: if err := WaitForPipelineRunState(ctx, c, prName, timeout, Chain( FailedWithReason(pod.ReasonCouldntGetTask, prName), - FailedWithMessage("requested resource 'https://artifacthub.io/api/v1/packages/tekton-task/tekton-catalog-tasks/git-clone-this-does-not-exist/0.7.0' not found on hub", prName), + FailedWithMessage("fail to fetch Artifact Hub resource: requested resource 'https://artifacthub.io/api/v1/packages/tekton-task/tekton-catalog-tasks/git-clone-this-does-not-exist' not found on hub", prName), ), "PipelineRunFailed", v1Version); err != nil { t.Fatalf("Error waiting for PipelineRun to finish with expected error: %s", err) }