From 6c85bffa950660744fad34872fbadfee79af914f Mon Sep 17 00:00:00 2001 From: Piyush Garg Date: Mon, 16 Sep 2024 01:25:14 +0530 Subject: [PATCH] Add support for multiple git resolver configurations This will add support for providing multiple configurations in git resolver configmap Configurations can be provided using identifier and identifier can be passed as a param in the taskrun/pipelinerun to resolver Old format is supported to provide backward compatibility and docs added in details about how to use the feature. Unit and e2e test added Fixes #5487 --- docs/git-resolver.md | 111 +++++++++++++++++- .../resolver/framework/fakeresolver.go | 4 +- .../resolver/framework/reconciler.go | 7 +- pkg/remoteresolution/resolver/git/resolver.go | 13 +- .../resolver/git/resolver_test.go | 88 ++++++++++++-- .../resolver/framework/fakeresolver.go | 4 +- .../resolver/framework/interface.go | 2 +- .../resolver/framework/reconciler.go | 7 +- pkg/resolution/resolver/git/config.go | 61 ++++++++++ pkg/resolution/resolver/git/params.go | 2 + pkg/resolution/resolver/git/resolver.go | 91 +++++++++----- pkg/resolution/resolver/git/resolver_test.go | 88 ++++++++++++-- test/resolvers_test.go | 64 ++++++++++ 13 files changed, 482 insertions(+), 60 deletions(-) diff --git a/docs/git-resolver.md b/docs/git-resolver.md index 7ec56fda7da..23d97e6ecd0 100644 --- a/docs/git-resolver.md +++ b/docs/git-resolver.md @@ -114,10 +114,6 @@ Note that not all `go-scm` implementations have been tested with the `git` resol * BitBucket Server * BitBucket Cloud -Fetching from multiple Git providers with different configuration is not -supported. You can use the [http resolver](./http-resolver.md) to fetch URL -from another provider with different credentials. - #### Task Resolution ```yaml @@ -195,6 +191,113 @@ spec: value: Ranni ``` +### Specifying Configuration for Multiple Git Providers + +You can specify configurations for multiple providers and even multiple configurations for same provider to use in +different tekton resources. You need to pass the config key mentioned in configmap as an extra param with key +`configKey`. If no identifier passed, `default` value will be considered. You can specify default configuration +in configmap either by mentioning no identifier or by using default identifier `default` + +### Sample Configmap + +You can add multiple configuration to `git-resolver-configmap` like this. All keys mentioned above are supported. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: git-resolver-config + namespace: tekton-pipelines-resolvers + labels: + app.kubernetes.io/component: resolvers + app.kubernetes.io/instance: default + app.kubernetes.io/part-of: tekton-pipelines +data: + # configuration 1, default one to use if no configKey provided or provided with value default + fetch-timeout: "1m" + default-url: "https://github.com/tektoncd/catalog.git" + default-revision: "main" + scm-type: "github" + server-url: "" + api-token-secret-name: "" + api-token-secret-key: "" + api-token-secret-namespace: "default" + default-org: "" + + # configuration 2, will be used if configKey param passed with value test1 + test1.fetch-timeout: "5m" + test1.default-url: "" + test1.default-revision: "stable" + test1.scm-type: "github" + test1.server-url: "api.internal-github.com" + test1.api-token-secret-name: "test1-secret" + test1.api-token-secret-key: "token" + test1.api-token-secret-namespace: "test1" + test1.default-org: "tektoncd" + + # configuration 3, will be used if configKey param passed with value test2 + test2.fetch-timeout: "10m" + test2.default-url: "" + test2.default-revision: "stable" + test2.scm-type: "gitlab" + test2.server-url: "api.internal-gitlab.com" + test2.api-token-secret-name: "test2-secret" + test2.api-token-secret-key: "pat" + test2.api-token-secret-namespace: "test2" + test2.default-org: "tektoncd-infra" +``` + +#### Task Resolution + +You can pass one more param `configKy` with value mentioned in configmap as identifier. + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: git-api-demo-tr +spec: + taskRef: + resolver: git + params: + - name: org + value: tektoncd + - name: repo + value: catalog + - name: revision + value: main + - name: pathInRepo + value: task/git-clone/0.6/git-clone.yaml + - name: configKey + value: test1 +``` + +#### Pipeline resolution + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: git-api-demo-pr +spec: + pipelineRef: + resolver: git + params: + - name: org + value: tektoncd + - name: repo + value: catalog + - name: revision + value: main + - name: pathInRepo + value: pipeline/simple/0.1/simple.yaml + - name: configKey + value: test2 + params: + - name: name + value: Ranni +``` + ## `ResolutionRequest` Status `ResolutionRequest.Status.RefSource` field captures the source where the remote resource came from. It includes the 3 subfields: `url`, `digest` and `entrypoint`. - `url` diff --git a/pkg/remoteresolution/resolver/framework/fakeresolver.go b/pkg/remoteresolution/resolver/framework/fakeresolver.go index ad7da0e6b85..046ec12f740 100644 --- a/pkg/remoteresolution/resolver/framework/fakeresolver.go +++ b/pkg/remoteresolution/resolver/framework/fakeresolver.go @@ -84,6 +84,6 @@ func (r *FakeResolver) Resolve(_ context.Context, req *v1beta1.ResolutionRequest var _ framework.TimedResolution = &FakeResolver{} // GetResolutionTimeout returns the configured timeout for the reconciler, or the default time.Duration if not configured. -func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - return framework.GetResolutionTimeout(r.Timeout, defaultTimeout) +func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + return framework.GetResolutionTimeout(r.Timeout, defaultTimeout), nil } diff --git a/pkg/remoteresolution/resolver/framework/reconciler.go b/pkg/remoteresolution/resolver/framework/reconciler.go index 96a3d223132..db334d514af 100644 --- a/pkg/remoteresolution/resolver/framework/reconciler.go +++ b/pkg/remoteresolution/resolver/framework/reconciler.go @@ -118,9 +118,14 @@ func (r *Reconciler) resolve(ctx context.Context, key string, rr *v1beta1.Resolu errChan := make(chan error) resourceChan := make(chan framework.ResolvedResource) + paramsMap := make(map[string]string) + for _, p := range rr.Spec.Params { + paramsMap[p.Name] = p.Value.StringVal + } + timeoutDuration := defaultMaximumResolutionDuration if timed, ok := r.resolver.(framework.TimedResolution); ok { - timeoutDuration = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration) + timeoutDuration, _ = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration, paramsMap) } // A new context is created for resolution so that timeouts can diff --git a/pkg/remoteresolution/resolver/git/resolver.go b/pkg/remoteresolution/resolver/git/resolver.go index 7231f4b007c..bdbe4eb1b58 100644 --- a/pkg/remoteresolution/resolver/git/resolver.go +++ b/pkg/remoteresolution/resolver/git/resolver.go @@ -141,13 +141,16 @@ var _ resolutionframework.TimedResolution = &Resolver{} // GetResolutionTimeout returns a time.Duration for the amount of time a // single git fetch may take. This can be configured with the // fetch-timeout field in the git-resolver-config configmap. -func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - conf := resolutionframework.GetResolverConfigFromContext(ctx) - if timeoutString, ok := conf[git.DefaultTimeoutKey]; ok { +func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + conf, err := git.GetScmConfigForParam(ctx, params) + if err != nil { + return time.Duration(0), err + } + if timeoutString := conf.Timeout; timeoutString != "" { timeout, err := time.ParseDuration(timeoutString) if err == nil { - return timeout + return timeout, nil } } - return defaultTimeout + return defaultTimeout, nil } diff --git a/pkg/remoteresolution/resolver/git/resolver_test.go b/pkg/remoteresolution/resolver/git/resolver_test.go index 18c3e629e70..c1b043983c7 100644 --- a/pkg/remoteresolution/resolver/git/resolver_test.go +++ b/pkg/remoteresolution/resolver/git/resolver_test.go @@ -219,7 +219,10 @@ func TestValidateParams_Failure(t *testing.T) { func TestGetResolutionTimeoutDefault(t *testing.T) { resolver := Resolver{} defaultTimeout := 30 * time.Minute - timeout := resolver.GetResolutionTimeout(context.Background(), defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(context.Background(), defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != defaultTimeout { t.Fatalf("expected default timeout to be returned") } @@ -233,12 +236,34 @@ func TestGetResolutionTimeoutCustom(t *testing.T) { gitresolution.DefaultTimeoutKey: configTimeout.String(), } ctx := resolutionframework.InjectResolverConfigToContext(context.Background(), config) - timeout := resolver.GetResolutionTimeout(ctx, defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != configTimeout { t.Fatalf("expected timeout from config to be returned") } } +func TestGetResolutionTimeoutCustomIdentifier(t *testing.T) { + resolver := Resolver{} + defaultTimeout := 30 * time.Minute + configTimeout := 5 * time.Second + identifierConfigTImeout := 10 * time.Second + config := map[string]string{ + gitresolution.DefaultTimeoutKey: configTimeout.String(), + "foo." + gitresolution.DefaultTimeoutKey: identifierConfigTImeout.String(), + } + ctx := resolutionframework.InjectResolverConfigToContext(context.Background(), config) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{"configKey": "foo"}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } + if timeout != identifierConfigTImeout { + t.Fatalf("expected timeout from config to be returned") + } +} + func TestResolveNotEnabled(t *testing.T) { resolver := Resolver{} @@ -268,6 +293,7 @@ type params struct { namespace string serverURL string scmType string + configKey string } func TestResolve(t *testing.T) { @@ -344,6 +370,7 @@ func TestResolve(t *testing.T) { expectedCommitSHA string expectedStatus *v1beta1.ResolutionRequestStatus expectedErr error + configIdentifer string }{{ name: "clone: default revision main", args: ¶ms{ @@ -439,6 +466,46 @@ func TestResolve(t *testing.T) { apiToken: "some-token", expectedCommitSHA: commitSHAsInSCMRepo[0], expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task from params api information with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + token: "token-secret", + tokenKey: "token", + namespace: "foo", + configKey: "test", + }, + config: map[string]string{ + "test." + gitresolution.ServerURLKey: "fake", + "test." + gitresolution.SCMTypeKey: "fake", + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + configKey: "test", + }, + config: map[string]string{ + "test." + gitresolution.ServerURLKey: "fake", + "test." + gitresolution.SCMTypeKey: "fake", + "test." + gitresolution.APISecretNameKey: "token-secret", + "test." + gitresolution.APISecretKeyKey: "token", + "test." + gitresolution.APISecretNamespaceKey: system.Namespace(), + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), }, { name: "api: successful pipeline", args: ¶ms{ @@ -591,9 +658,9 @@ func TestResolve(t *testing.T) { if cfg == nil { cfg = make(map[string]string) } - cfg[gitresolution.DefaultTimeoutKey] = "1m" - if cfg[gitresolution.DefaultRevisionKey] == "" { - cfg[gitresolution.DefaultRevisionKey] = plumbing.Master.Short() + cfg[tc.configIdentifer+gitresolution.DefaultTimeoutKey] = "1m" + if cfg[tc.configIdentifer+gitresolution.DefaultRevisionKey] == "" { + cfg[tc.configIdentifer+gitresolution.DefaultRevisionKey] = plumbing.Master.Short() } request := createRequest(tc.args) @@ -654,8 +721,8 @@ func TestResolve(t *testing.T) { frtesting.RunResolverReconcileTest(ctx, t, d, resolver, request, expectedStatus, tc.expectedErr, func(resolver framework.Resolver, testAssets test.Assets) { var secretName, secretNameKey, secretNamespace string - if tc.config[gitresolution.APISecretNameKey] != "" && tc.config[gitresolution.APISecretNamespaceKey] != "" && tc.config[gitresolution.APISecretKeyKey] != "" && tc.apiToken != "" { - secretName, secretNameKey, secretNamespace = tc.config[gitresolution.APISecretNameKey], tc.config[gitresolution.APISecretKeyKey], tc.config[gitresolution.APISecretNamespaceKey] + if tc.config[tc.configIdentifer+gitresolution.APISecretNameKey] != "" && tc.config[tc.configIdentifer+gitresolution.APISecretNamespaceKey] != "" && tc.config[tc.configIdentifer+gitresolution.APISecretKeyKey] != "" && tc.apiToken != "" { + secretName, secretNameKey, secretNamespace = tc.config[tc.configIdentifer+gitresolution.APISecretNameKey], tc.config[tc.configIdentifer+gitresolution.APISecretKeyKey], tc.config[tc.configIdentifer+gitresolution.APISecretNamespaceKey] } if tc.args.token != "" && tc.args.namespace != "" && tc.args.tokenKey != "" { secretName, secretNameKey, secretNamespace = tc.args.token, tc.args.tokenKey, tc.args.namespace @@ -879,6 +946,13 @@ func createRequest(args *params) *v1beta1.ResolutionRequest { } } + if args.configKey != "" { + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: gitresolution.ConfigKeyParam, + Value: *pipelinev1.NewStructuredValues(args.configKey), + }) + } + return rr } diff --git a/pkg/resolution/resolver/framework/fakeresolver.go b/pkg/resolution/resolver/framework/fakeresolver.go index 3fd363f825c..b22349d6f56 100644 --- a/pkg/resolution/resolver/framework/fakeresolver.go +++ b/pkg/resolution/resolver/framework/fakeresolver.go @@ -166,8 +166,8 @@ func Resolve(params []pipelinev1.Param, forParam map[string]*FakeResolvedResourc var _ TimedResolution = &FakeResolver{} // GetResolutionTimeout returns the configured timeout for the reconciler, or the default time.Duration if not configured. -func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - return GetResolutionTimeout(r.Timeout, defaultTimeout) +func (r *FakeResolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + return GetResolutionTimeout(r.Timeout, defaultTimeout), nil } // GetResolutionTimeout returns the input timeout if set to something greater than 0 or the default time.Duration if not configured. diff --git a/pkg/resolution/resolver/framework/interface.go b/pkg/resolution/resolver/framework/interface.go index 39e1fbfba06..89c1e897a77 100644 --- a/pkg/resolution/resolver/framework/interface.go +++ b/pkg/resolution/resolver/framework/interface.go @@ -90,7 +90,7 @@ type TimedResolution interface { // object, which includes any request-scoped data like // resolver config and the request's originating namespace, // along with a default. - GetResolutionTimeout(ctx context.Context, timeout time.Duration) time.Duration + GetResolutionTimeout(ctx context.Context, timeout time.Duration, params map[string]string) (time.Duration, error) } // ResolvedResource returns the data and annotations of a successful diff --git a/pkg/resolution/resolver/framework/reconciler.go b/pkg/resolution/resolver/framework/reconciler.go index 4950a90c74d..4340c7a9355 100644 --- a/pkg/resolution/resolver/framework/reconciler.go +++ b/pkg/resolution/resolver/framework/reconciler.go @@ -109,9 +109,14 @@ func (r *Reconciler) resolve(ctx context.Context, key string, rr *v1beta1.Resolu errChan := make(chan error) resourceChan := make(chan ResolvedResource) + paramsMap := make(map[string]string) + for _, p := range rr.Spec.Params { + paramsMap[p.Name] = p.Value.StringVal + } + timeoutDuration := defaultMaximumResolutionDuration if timed, ok := r.resolver.(TimedResolution); ok { - timeoutDuration = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration) + timeoutDuration, _ = timed.GetResolutionTimeout(ctx, defaultMaximumResolutionDuration, paramsMap) } // A new context is created for resolution so that timeouts can diff --git a/pkg/resolution/resolver/git/config.go b/pkg/resolution/resolver/git/config.go index 44645b1fae6..975166d637a 100644 --- a/pkg/resolution/resolver/git/config.go +++ b/pkg/resolution/resolver/git/config.go @@ -16,6 +16,15 @@ limitations under the License. package git +import ( + "context" + "fmt" + "reflect" + "strings" + + "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" +) + const ( // DefaultTimeoutKey is the configuration field name for controlling // the maximum duration of a resolution request for a file from git. @@ -43,3 +52,55 @@ const ( // APISecretNamespaceKey is the config map key for the token secret's namespace APISecretNamespaceKey = "api-token-secret-namespace" ) + +type GitResolverConfig map[string]ScmConfig + +type ScmConfig struct { + Timeout string `json:"fetch-timeout"` + URL string `json:"default-url"` + Revision string `json:"default-revision"` + Org string `json:"default-org"` + ServerURL string `json:"server-url"` + SCMType string `json:"scm-type"` + APISecretName string `json:"api-token-secret-name"` + APISecretKey string `json:"api-token-secret-key"` + APISecretNamespace string `json:"api-token-secret-namespace"` +} + +func GetGitResolverConfig(ctx context.Context) (GitResolverConfig, error) { + var scmConfig interface{} = &ScmConfig{} + structType := reflect.TypeOf(scmConfig).Elem() + gitResolverConfig := map[string]ScmConfig{} + conf := framework.GetResolverConfigFromContext(ctx) + for key, value := range conf { + var configIdentifier, configKey string + splittedKeyName := strings.Split(key, ".") + switch len(splittedKeyName) { + case 2: + configKey = splittedKeyName[1] + configIdentifier = splittedKeyName[0] + case 1: + configKey = key + configIdentifier = "default" + default: + return nil, fmt.Errorf("key %s passed in git resolver configmap is invalid", key) + } + _, ok := gitResolverConfig[configIdentifier] + if !ok { + gitResolverConfig[configIdentifier] = ScmConfig{} + } + for i := range structType.NumField() { + field := structType.Field(i) + fieldName := field.Name + jsonTag := field.Tag.Get("json") + if configKey == jsonTag { + tokenDetails := gitResolverConfig[configIdentifier] + var scm interface{} = &tokenDetails + structValue := reflect.ValueOf(scm).Elem() + structValue.FieldByName(fieldName).SetString(value) + gitResolverConfig[configIdentifier] = structValue.Interface().(ScmConfig) + } + } + } + return gitResolverConfig, nil +} diff --git a/pkg/resolution/resolver/git/params.go b/pkg/resolution/resolver/git/params.go index f1885f7c2b0..d7cd114d8d6 100644 --- a/pkg/resolution/resolver/git/params.go +++ b/pkg/resolution/resolver/git/params.go @@ -39,4 +39,6 @@ const ( ScmTypeParam string = "scmType" // serverURLParam is an optional string to the server URL for the SCM API to connect to ServerURLParam string = "serverURL" + // ConfigKeyParam is an optional string to provid which scm configuration to use from git resolver configmap + ConfigKeyParam string = "configKey" ) diff --git a/pkg/resolution/resolver/git/resolver.go b/pkg/resolution/resolver/git/resolver.go index b0226e2bebc..539082474b7 100644 --- a/pkg/resolution/resolver/git/resolver.go +++ b/pkg/resolution/resolver/git/resolver.go @@ -158,20 +158,21 @@ func validateRepoURL(url string) bool { } func ResolveAnonymousGit(ctx context.Context, params map[string]string) (framework.ResolvedResource, error) { - conf := framework.GetResolverConfigFromContext(ctx) + conf, err := GetScmConfigForParam(ctx, params) + if err != nil { + return nil, err + } repo := params[UrlParam] if repo == "" { - if urlString, ok := conf[DefaultURLKey]; ok { - repo = urlString - } else { + urlString := conf.URL + if urlString == "" { return nil, errors.New("default Git Repo Url was not set during installation of the git resolver") } } revision := params[RevisionParam] if revision == "" { - if revisionString, ok := conf[DefaultRevisionKey]; ok { - revision = revisionString - } else { + revisionString := conf.Revision + if revisionString == "" { return nil, errors.New("default Git Revision was not set during installation of the git resolver") } } @@ -247,29 +248,36 @@ var _ framework.TimedResolution = &Resolver{} // GetResolutionTimeout returns a time.Duration for the amount of time a // single git fetch may take. This can be configured with the // fetch-timeout field in the git-resolver-config configmap. -func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration) time.Duration { - conf := framework.GetResolverConfigFromContext(ctx) - if timeoutString, ok := conf[DefaultTimeoutKey]; ok { +func (r *Resolver) GetResolutionTimeout(ctx context.Context, defaultTimeout time.Duration, params map[string]string) (time.Duration, error) { + conf, err := GetScmConfigForParam(ctx, params) + if err != nil { + return time.Duration(0), err + } + if timeoutString := conf.Timeout; timeoutString != "" { timeout, err := time.ParseDuration(timeoutString) if err == nil { - return timeout + return timeout, nil } } - return defaultTimeout + return defaultTimeout, nil } func PopulateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[string]string, error) { - conf := framework.GetResolverConfigFromContext(ctx) - paramsMap := make(map[string]string) for _, p := range params { paramsMap[p.Name] = p.Value.StringVal } + conf, err := GetScmConfigForParam(ctx, paramsMap) + if err != nil { + return nil, err + } + var missingParams []string if _, ok := paramsMap[RevisionParam]; !ok { - if defaultRevision, ok := conf[DefaultRevisionKey]; ok { + defaultRevision := conf.Revision + if defaultRevision != "" { paramsMap[RevisionParam] = defaultRevision } else { missingParams = append(missingParams, RevisionParam) @@ -284,7 +292,8 @@ func PopulateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[ } if paramsMap[UrlParam] == "" && paramsMap[RepoParam] == "" { - if urlString, ok := conf[DefaultURLKey]; ok { + urlString := conf.URL + if urlString != "" { paramsMap[UrlParam] = urlString } else { return nil, fmt.Errorf("must specify one of '%s' or '%s'", UrlParam, RepoParam) @@ -293,7 +302,8 @@ func PopulateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[ if paramsMap[RepoParam] != "" { if _, ok := paramsMap[OrgParam]; !ok { - if defaultOrg, ok := conf[DefaultOrgKey]; ok { + defaultOrg := conf.Org + if defaultOrg != "" { paramsMap[OrgParam] = defaultOrg } else { return nil, fmt.Errorf("'%s' is required when '%s' is specified", OrgParam, RepoParam) @@ -394,7 +404,7 @@ func ResolveAPIGit(ctx context.Context, params map[string]string, kubeclient kub } else { secretRef = nil } - apiToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl) + apiToken, err := getAPIToken(ctx, secretRef, kubeclient, logger, cache, ttl, params) if err != nil { return nil, err } @@ -438,8 +448,11 @@ func ResolveAPIGit(ctx context.Context, params map[string]string, kubeclient kub }, nil } -func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration) ([]byte, error) { - conf := framework.GetResolverConfigFromContext(ctx) +func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kubernetes.Interface, logger *zap.SugaredLogger, cache *cache.LRUExpireCache, ttl time.Duration, params map[string]string) ([]byte, error) { + conf, err := GetScmConfigForParam(ctx, params) + if err != nil { + return nil, err + } ok := false @@ -451,21 +464,24 @@ func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kube } if apiSecret.name == "" { - if apiSecret.name, ok = conf[APISecretNameKey]; !ok || apiSecret.name == "" { + apiSecret.name = conf.APISecretName + if apiSecret.name == "" { err := fmt.Errorf("cannot get API token, required when specifying '%s' param, '%s' not specified in config", RepoParam, APISecretNameKey) logger.Info(err) return nil, err } } if apiSecret.key == "" { - if apiSecret.key, ok = conf[APISecretKeyKey]; !ok || apiSecret.key == "" { + apiSecret.key = conf.APISecretKey + if apiSecret.key == "" { err := fmt.Errorf("cannot get API token, required when specifying '%s' param, '%s' not specified in config", RepoParam, APISecretKeyKey) logger.Info(err) return nil, err } } if apiSecret.ns == "" { - if apiSecret.ns, ok = conf[APISecretNamespaceKey]; !ok { + apiSecret.ns = conf.APISecretNamespace + if apiSecret.ns == "" { apiSecret.ns = os.Getenv("SYSTEM_NAMESPACE") } } @@ -502,16 +518,18 @@ func getAPIToken(ctx context.Context, apiSecret *secretCacheKey, kubeclient kube } func getSCMTypeAndServerURL(ctx context.Context, params map[string]string) (string, string, error) { - conf := framework.GetResolverConfigFromContext(ctx) + conf, err := GetScmConfigForParam(ctx, params) + if err != nil { + return "", "", err + } var scmType, serverURL string if key, ok := params[ScmTypeParam]; ok { scmType = key } if scmType == "" { - if key, ok := conf[SCMTypeKey]; ok && scmType == "" { - scmType = key - } else { + scmType = conf.SCMType + if scmType == "" { return "", "", fmt.Errorf("missing or empty %s value in configmap", SCMTypeKey) } } @@ -519,9 +537,8 @@ func getSCMTypeAndServerURL(ctx context.Context, params map[string]string) (stri serverURL = key } if serverURL == "" { - if key, ok := conf[ServerURLKey]; ok && serverURL == "" { - serverURL = key - } else { + serverURL = conf.ServerURL + if serverURL == "" { return "", "", fmt.Errorf("missing or empty %s value in configmap", ServerURLKey) } } @@ -532,3 +549,17 @@ func IsDisabled(ctx context.Context) bool { cfg := resolverconfig.FromContextOrDefaults(ctx) return !cfg.FeatureFlags.EnableGitResolver } + +func GetScmConfigForParam(ctx context.Context, params map[string]string) (ScmConfig, error) { + gitResolverConfig, err := GetGitResolverConfig(ctx) + if err != nil { + return ScmConfig{}, err + } + if configKeyToUse, ok := params[ConfigKeyParam]; ok { + if config, exist := gitResolverConfig[configKeyToUse]; exist { + return config, nil + } + return ScmConfig{}, fmt.Errorf("no git resolver configuration found for configKey %s", configKeyToUse) + } + return gitResolverConfig["default"], nil +} diff --git a/pkg/resolution/resolver/git/resolver_test.go b/pkg/resolution/resolver/git/resolver_test.go index 7185abfe6aa..41722c52acc 100644 --- a/pkg/resolution/resolver/git/resolver_test.go +++ b/pkg/resolution/resolver/git/resolver_test.go @@ -216,7 +216,10 @@ func TestValidateParams_Failure(t *testing.T) { func TestGetResolutionTimeoutDefault(t *testing.T) { resolver := Resolver{} defaultTimeout := 30 * time.Minute - timeout := resolver.GetResolutionTimeout(context.Background(), defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(context.Background(), defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != defaultTimeout { t.Fatalf("expected default timeout to be returned") } @@ -230,12 +233,34 @@ func TestGetResolutionTimeoutCustom(t *testing.T) { DefaultTimeoutKey: configTimeout.String(), } ctx := framework.InjectResolverConfigToContext(context.Background(), config) - timeout := resolver.GetResolutionTimeout(ctx, defaultTimeout) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } if timeout != configTimeout { t.Fatalf("expected timeout from config to be returned") } } +func TestGetResolutionTimeoutCustomIdentifier(t *testing.T) { + resolver := Resolver{} + defaultTimeout := 30 * time.Minute + configTimeout := 5 * time.Second + identifierConfigTImeout := 10 * time.Second + config := map[string]string{ + DefaultTimeoutKey: configTimeout.String(), + "foo." + DefaultTimeoutKey: identifierConfigTImeout.String(), + } + ctx := framework.InjectResolverConfigToContext(context.Background(), config) + timeout, err := resolver.GetResolutionTimeout(ctx, defaultTimeout, map[string]string{"configKey": "foo"}) + if err != nil { + t.Fatalf("couldn't get default-timeout: %v", err) + } + if timeout != identifierConfigTImeout { + t.Fatalf("expected timeout from config to be returned") + } +} + func TestResolveNotEnabled(t *testing.T) { resolver := Resolver{} @@ -265,6 +290,7 @@ type params struct { namespace string serverURL string scmType string + configKey string } func TestResolve(t *testing.T) { @@ -341,6 +367,7 @@ func TestResolve(t *testing.T) { expectedCommitSHA string expectedStatus *v1beta1.ResolutionRequestStatus expectedErr error + configIdentifer string }{{ name: "clone: default revision main", args: ¶ms{ @@ -436,6 +463,46 @@ func TestResolve(t *testing.T) { apiToken: "some-token", expectedCommitSHA: commitSHAsInSCMRepo[0], expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task from params api information with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + token: "token-secret", + tokenKey: "token", + namespace: "foo", + configKey: "test", + }, + config: map[string]string{ + "test." + ServerURLKey: "fake", + "test." + SCMTypeKey: "fake", + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), + }, { + name: "api: successful task with identifier", + args: ¶ms{ + revision: "main", + pathInRepo: "tasks/example-task.yaml", + org: testOrg, + repo: testRepo, + configKey: "test", + }, + config: map[string]string{ + "test." + ServerURLKey: "fake", + "test." + SCMTypeKey: "fake", + "test." + APISecretNameKey: "token-secret", + "test." + APISecretKeyKey: "token", + "test." + APISecretNamespaceKey: system.Namespace(), + }, + configIdentifer: "test.", + apiToken: "some-token", + expectedCommitSHA: commitSHAsInSCMRepo[0], + expectedStatus: resolution.CreateResolutionRequestStatusWithData(mainTaskYAML), }, { name: "api: successful pipeline", args: ¶ms{ @@ -588,9 +655,9 @@ func TestResolve(t *testing.T) { if cfg == nil { cfg = make(map[string]string) } - cfg[DefaultTimeoutKey] = "1m" - if cfg[DefaultRevisionKey] == "" { - cfg[DefaultRevisionKey] = plumbing.Master.Short() + cfg[tc.configIdentifer+DefaultTimeoutKey] = "1m" + if cfg[tc.configIdentifer+DefaultRevisionKey] == "" { + cfg[tc.configIdentifer+DefaultRevisionKey] = plumbing.Master.Short() } request := createRequest(tc.args) @@ -651,8 +718,8 @@ func TestResolve(t *testing.T) { frtesting.RunResolverReconcileTest(ctx, t, d, resolver, request, expectedStatus, tc.expectedErr, func(resolver framework.Resolver, testAssets test.Assets) { var secretName, secretNameKey, secretNamespace string - if tc.config[APISecretNameKey] != "" && tc.config[APISecretNamespaceKey] != "" && tc.config[APISecretKeyKey] != "" && tc.apiToken != "" { - secretName, secretNameKey, secretNamespace = tc.config[APISecretNameKey], tc.config[APISecretKeyKey], tc.config[APISecretNamespaceKey] + if tc.config[tc.configIdentifer+APISecretNameKey] != "" && tc.config[tc.configIdentifer+APISecretNamespaceKey] != "" && tc.config[tc.configIdentifer+APISecretKeyKey] != "" && tc.apiToken != "" { + secretName, secretNameKey, secretNamespace = tc.config[tc.configIdentifer+APISecretNameKey], tc.config[tc.configIdentifer+APISecretKeyKey], tc.config[tc.configIdentifer+APISecretNamespaceKey] } if tc.args.token != "" && tc.args.namespace != "" && tc.args.tokenKey != "" { secretName, secretNameKey, secretNamespace = tc.args.token, tc.args.tokenKey, tc.args.namespace @@ -876,6 +943,13 @@ func createRequest(args *params) *v1beta1.ResolutionRequest { } } + if args.configKey != "" { + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: ConfigKeyParam, + Value: *pipelinev1.NewStructuredValues(args.configKey), + }) + } + return rr } diff --git a/test/resolvers_test.go b/test/resolvers_test.go index 2b3bd5a4ae3..813e6b34d47 100644 --- a/test/resolvers_test.go +++ b/test/resolvers_test.go @@ -533,6 +533,70 @@ spec: } } +func TestGitResolver_API_Identifier(t *testing.T) { + ctx := context.Background() + c, namespace := setup(ctx, t, gitFeatureFlags) + + t.Parallel() + + knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf) + defer tearDown(ctx, t, c, namespace) + + giteaClusterHostname, tokenSecretName := setupGitea(ctx, t, c, namespace) + + resovlerNS := resolverconfig.ResolversNamespace(systemNamespace) + + originalConfigMap, err := c.KubeClient.CoreV1().ConfigMaps(resovlerNS).Get(ctx, gitresolution.ConfigMapName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Failed to get ConfigMap `%s`: %s", gitresolution.ConfigMapName, err) + } + originalConfigMapData := originalConfigMap.Data + + t.Logf("Creating ConfigMap %s", gitresolution.ConfigMapName) + configMapData := map[string]string{ + "test." + gitresolution.ServerURLKey: fmt.Sprint("http://", net.JoinHostPort(giteaClusterHostname, "3000")), + "test." + gitresolution.SCMTypeKey: "gitea", + "test." + gitresolution.APISecretNameKey: tokenSecretName, + "test." + gitresolution.APISecretKeyKey: scmTokenSecretKey, + "test." + gitresolution.APISecretNamespaceKey: namespace, + } + if err := updateConfigMap(ctx, c.KubeClient, resovlerNS, gitresolution.ConfigMapName, configMapData); err != nil { + t.Fatal(err) + } + defer resetConfigMap(ctx, t, c, resovlerNS, gitresolution.ConfigMapName, originalConfigMapData) + + trName := helpers.ObjectNameForTest(t) + tr := parse.MustParseV1TaskRun(t, fmt.Sprintf(` +metadata: + name: %s + namespace: %s +spec: + taskRef: + resolver: git + params: + - name: revision + value: %s + - name: pathInRepo + value: %s + - name: org + value: %s + - name: repo + value: %s + - name: configKey + value: test +`, trName, namespace, scmRemoteBranch, scmRemoteTaskPath, scmRemoteOrg, scmRemoteRepo)) + + _, err = c.V1TaskRunClient.Create(ctx, tr, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("Failed to create TaskRun: %v", err) + } + + t.Logf("Waiting for TaskRun %s in namespace %s to complete", trName, namespace) + if err := WaitForTaskRunState(ctx, c, trName, TaskRunSucceed(trName), "TaskRunSuccess", v1Version); err != nil { + t.Fatalf("Error waiting for TaskRun %s to finish: %s", trName, err) + } +} + // setupGitea reads git-resolver/gitea.yaml, replaces "default" namespace references in "namespace: default" and // svc.cluster.local hostnames with the test namespace, calls kubectl create, and waits for the gitea-0 pod to be up // and running. At that point, it'll create a test user and token, create a Secret containing that token, create an org