From 724f260e499c054a40165785c75519bc6b3dfb21 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 27 Oct 2023 16:33:08 +0200 Subject: [PATCH] Add credentials to HTTP resolver This adds the ability to pass credentials to the HTTP resolver when fetching the URL. This let's for example to fetch tasks from SCM private repositories on other SCM providers than configured with the git resolver. Fixes #7296 Signed-off-by: Chmouel Boudjnah --- docs/git-resolver.md | 4 + docs/http-resolver.md | 30 +- .../beta/http-resolver-credentials.yaml | 35 +++ .../framework/testing/fakecontroller.go | 6 +- pkg/resolution/resolver/http/params.go | 13 +- pkg/resolution/resolver/http/resolver.go | 97 ++++++- pkg/resolution/resolver/http/resolver_test.go | 274 +++++++++++++++++- 7 files changed, 432 insertions(+), 27 deletions(-) create mode 100644 examples/v1/pipelineruns/beta/http-resolver-credentials.yaml diff --git a/docs/git-resolver.md b/docs/git-resolver.md index 463922dab7c..0f659cfa969 100644 --- a/docs/git-resolver.md +++ b/docs/git-resolver.md @@ -112,6 +112,10 @@ Note that not all `go-scm` implementations have been tested with the `git` resol * BitBucket Server * BitBucket Cloud +Fetching from multiple Git providers with different configuration is not +supported. You can use the [http resolver](./http-resolver.md) to fetch URL +from another provider with different credentials. + #### Task Resolution ```yaml diff --git a/docs/http-resolver.md b/docs/http-resolver.md index 8dada19a58c..85cdd96959c 100644 --- a/docs/http-resolver.md +++ b/docs/http-resolver.md @@ -11,9 +11,12 @@ This resolver responds to type `http`. ## Parameters -| Param Name | Description | Example Value | -|------------------|-------------------------------------------------------------------------------|------------------------------------------------------------| -| `url` | The URL to fetch from | https://raw.githubusercontent.com/tektoncd-catalog/git-clone/main/task/git-clone/git-clone.yaml | +| Param Name | Description | Example Value | | +|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------|---| +| `url` | The URL to fetch from | https://raw.githubusercontent.com/tektoncd-catalog/git-clone/main/task/git-clone/git-clone.yaml | | +| `http-username` | An optional username when fetching a task with credentials (need to be used in conjunction with `http-password-secret`) | `git` | | +| `http-password-secret` | An optional secret in the PipelineRun directory with a reference to a password when fetching a task with credentials (need to be used in conjunction with `http-username`) | `http-password` | | +| `http-password-secret-key` | An optional key in the `http-password-secret` to be used when fetching a task with credentials | Default: `password` | | A valid URL must be provided. Only HTTP or HTTPS URLs are supported. @@ -54,6 +57,27 @@ spec: value: https://raw.githubusercontent.com/tektoncd-catalog/git-clone/main/task/git-clone/git-clone.yaml ``` +### Task Resolution with Basic Auth + +```yaml +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + name: remote-task-reference +spec: + taskRef: + resolver: http + params: + - name: url + value: https://raw.githubusercontent.com/owner/private-repo/main/task/task.yaml + - name: http-username + value: git + - name: http-password-secret + value: git-secret + - name: http-password-secret-key + value: git-token +``` + ### Pipeline Resolution ```yaml diff --git a/examples/v1/pipelineruns/beta/http-resolver-credentials.yaml b/examples/v1/pipelineruns/beta/http-resolver-credentials.yaml new file mode 100644 index 00000000000..632fe528f83 --- /dev/null +++ b/examples/v1/pipelineruns/beta/http-resolver-credentials.yaml @@ -0,0 +1,35 @@ +# This http resolver example will uses a username and password to access the +# URL. +# +# http-password-secret is a Kubernetes secret containing the +# password in the same namespace where this PipelineRun runs. +--- +kind: Secret +apiVersion: v1 +metadata: + name: my-secret +stringData: + token: "token" +--- +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: http-resolver- +spec: + pipelineSpec: + tasks: + - name: http-resolver + taskRef: + resolver: http + params: + - name: url + value: https://api.hub.tekton.dev/v1/resource/tekton/task/tkn/0.4/raw + - name: http-username + value: git + - name: http-password-secret + value: my-secret + - name: http-password-secret-key + value: token + params: + - name: ARGS + value: ["version"] diff --git a/pkg/resolution/resolver/framework/testing/fakecontroller.go b/pkg/resolution/resolver/framework/testing/fakecontroller.go index b15e823930b..4ddc3a8c95c 100644 --- a/pkg/resolution/resolver/framework/testing/fakecontroller.go +++ b/pkg/resolution/resolver/framework/testing/fakecontroller.go @@ -71,14 +71,14 @@ func RunResolverReconcileTest(ctx context.Context, t *testing.T, d test.Data, re err := testAssets.Controller.Reconciler.Reconcile(testAssets.Ctx, getRequestName(request)) if expectedErr != nil { if err == nil { - t.Fatalf("expected to get error %v, but got nothing", expectedErr) + t.Fatalf("expected to get error: `%v`, but got nothing", expectedErr) } if expectedErr.Error() != err.Error() { - t.Fatalf("expected to get error %v, but got %v", expectedErr, err) + t.Fatalf("expected to get error `%v`, but got `%v`", expectedErr, err) } } else if err != nil { if ok, _ := controller.IsRequeueKey(err); !ok { - t.Fatalf("did not expect an error, but got %v", err) + t.Fatalf("did not expect an error, but got `%v`", err) } } diff --git a/pkg/resolution/resolver/http/params.go b/pkg/resolution/resolver/http/params.go index c997a90d05a..1a2f9bd7821 100644 --- a/pkg/resolution/resolver/http/params.go +++ b/pkg/resolution/resolver/http/params.go @@ -14,6 +14,15 @@ limitations under the License. package http const ( - // urlParam is the url to fetch the task from - urlParam string = "url" + // urlParamType is the URL to fetch the task from + urlParamType string = "url" + + // httpBasicAuthUsername is the user name to use for basic auth + httpBasicAuthUsername string = "http-username" + + // httpBasicAuthSecret is the reference to a secret in the PipelineRun namespace to use for basic auth + httpBasicAuthSecret string = "http-password-secret" + + // httpBasicAuthSecretKey is the key in the httpBasicAuthSecret secret to use for basic auth + httpBasicAuthSecretKey string = "http-password-secret-key" ) diff --git a/pkg/resolution/resolver/http/resolver.go b/pkg/resolution/resolver/http/resolver.go index c73133cb582..50911b90b4c 100644 --- a/pkg/resolution/resolver/http/resolver.go +++ b/pkg/resolution/resolver/http/resolver.go @@ -16,6 +16,7 @@ package http import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" "fmt" @@ -29,6 +30,12 @@ import ( pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" "github.com/tektoncd/pipeline/pkg/resolution/common" "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" + "go.uber.org/zap" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + kubeclient "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/logging" ) const ( @@ -41,17 +48,25 @@ const ( // httpResolverName The name of the resolver httpResolverName = "Http" - // ConfigMapName is the http resolver's config map + // configMapName is the http resolver's config map configMapName = "http-resolver-config" // default Timeout value when fetching http resources in seconds defaultHttpTimeoutValue = "1m" + + // default key in the HTTP password secret + defaultBasicAuthSecretKey = "password" ) // Resolver implements a framework.Resolver that can fetch files from an HTTP URL -type Resolver struct{} +type Resolver struct { + kubeClient kubernetes.Interface + logger *zap.SugaredLogger +} -func (r *Resolver) Initialize(context.Context) error { +func (r *Resolver) Initialize(ctx context.Context) error { + r.kubeClient = kubeclient.Get(ctx) + r.logger = logging.FromContext(ctx) return nil } @@ -95,7 +110,7 @@ func (r *Resolver) Resolve(ctx context.Context, oParams []pipelinev1.Param) (fra return nil, err } - return fetchHttpResource(ctx, params) + return r.fetchHttpResource(ctx, params) } func (r *Resolver) isDisabled(ctx context.Context) bool { @@ -144,15 +159,33 @@ func populateDefaultParams(ctx context.Context, params []pipelinev1.Param) (map[ var missingParams []string - if _, ok := paramsMap[urlParam]; !ok { - missingParams = append(missingParams, urlParam) + if _, ok := paramsMap[urlParamType]; !ok { + missingParams = append(missingParams, urlParamType) } else { - u, err := url.ParseRequestURI(paramsMap[urlParam]) + u, err := url.ParseRequestURI(paramsMap[urlParamType]) if err != nil { - return nil, fmt.Errorf("cannot parse url %s: %w", paramsMap[urlParam], err) + return nil, fmt.Errorf("cannot parse url %s: %w", paramsMap[urlParamType], err) } if u.Scheme != "http" && u.Scheme != "https" { - return nil, fmt.Errorf("url %s is not a valid http(s) url", paramsMap[urlParam]) + return nil, fmt.Errorf("url %s is not a valid http(s) url", paramsMap[urlParamType]) + } + } + + if username, ok := paramsMap[httpBasicAuthUsername]; ok { + if _, ok := paramsMap[httpBasicAuthSecret]; !ok { + return nil, fmt.Errorf("missing required param %s when using %s", httpBasicAuthSecret, httpBasicAuthUsername) + } + if username == "" { + return nil, fmt.Errorf("value %s cannot be empty", httpBasicAuthUsername) + } + } + + if secret, ok := paramsMap[httpBasicAuthSecret]; ok { + if _, ok := paramsMap[httpBasicAuthUsername]; !ok { + return nil, fmt.Errorf("missing required param %s when using %s", httpBasicAuthUsername, httpBasicAuthSecret) + } + if secret == "" { + return nil, fmt.Errorf("value %s cannot be empty", httpBasicAuthSecret) } } @@ -178,7 +211,7 @@ func makeHttpClient(ctx context.Context) (*http.Client, error) { }, nil } -func fetchHttpResource(ctx context.Context, params map[string]string) (framework.ResolvedResource, error) { +func (r *Resolver) fetchHttpResource(ctx context.Context, params map[string]string) (framework.ResolvedResource, error) { var targetURL string var ok bool @@ -187,8 +220,8 @@ func fetchHttpResource(ctx context.Context, params map[string]string) (framework return nil, err } - if targetURL, ok = params[urlParam]; !ok { - return nil, fmt.Errorf("missing required params: %s", urlParam) + if targetURL, ok = params[urlParamType]; !ok { + return nil, fmt.Errorf("missing required params: %s", urlParamType) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) @@ -196,6 +229,15 @@ func fetchHttpResource(ctx context.Context, params map[string]string) (framework return nil, fmt.Errorf("constructing request: %w", err) } + // NOTE(chmouel): We already made sure that username and secret was specified by the user + if secret, ok := params[httpBasicAuthSecret]; ok && secret != "" { + if encodedSecret, err := r.getBasicAuthSecret(ctx, params); err != nil { + return nil, err + } else { + req.Header.Set("Authorization", encodedSecret) + } + } + resp, err := httpClient.Do(req) if err != nil { return nil, fmt.Errorf("error fetching URL: %w", err) @@ -216,3 +258,34 @@ func fetchHttpResource(ctx context.Context, params map[string]string) (framework URL: targetURL, }, nil } + +func (r *Resolver) getBasicAuthSecret(ctx context.Context, params map[string]string) (string, error) { + secretName := params[httpBasicAuthSecret] + userName := params[httpBasicAuthUsername] + tokenSecretKey := defaultBasicAuthSecretKey + if v, ok := params[httpBasicAuthSecretKey]; ok { + if v != "" { + tokenSecretKey = v + } + } + secretNS := common.RequestNamespace(ctx) + secret, err := r.kubeClient.CoreV1().Secrets(secretNS).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + notFoundErr := fmt.Errorf("cannot get API token, secret %s not found in namespace %s", secretName, secretNS) + r.logger.Info(notFoundErr) + return "", notFoundErr + } + wrappedErr := fmt.Errorf("error reading API token from secret %s in namespace %s: %w", secretName, secretNS, err) + r.logger.Info(wrappedErr) + return "", wrappedErr + } + secretVal, ok := secret.Data[tokenSecretKey] + if !ok { + err := fmt.Errorf("cannot get API token, key %s not found in secret %s in namespace %s", tokenSecretKey, secretName, secretNS) + r.logger.Info(err) + return "", err + } + return fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString( + []byte(fmt.Sprintf("%s:%s", userName, secretVal)))), nil +} diff --git a/pkg/resolution/resolver/http/resolver_test.go b/pkg/resolution/resolver/http/resolver_test.go index 469327017a2..bad93792319 100644 --- a/pkg/resolution/resolver/http/resolver_test.go +++ b/pkg/resolution/resolver/http/resolver_test.go @@ -19,21 +19,52 @@ package http import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" + "errors" "fmt" "net/http" "net/http/httptest" "regexp" "testing" + "time" "github.com/google/go-cmp/cmp" + resolverconfig "github.com/tektoncd/pipeline/pkg/apis/config/resolver" pipelinev1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + "github.com/tektoncd/pipeline/pkg/apis/resolution/v1beta1" + ttesting "github.com/tektoncd/pipeline/pkg/reconciler/testing" resolutioncommon "github.com/tektoncd/pipeline/pkg/resolution/common" "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework" frtesting "github.com/tektoncd/pipeline/pkg/resolution/resolver/framework/testing" + "github.com/tektoncd/pipeline/pkg/resolution/resolver/internal" + "github.com/tektoncd/pipeline/test" "github.com/tektoncd/pipeline/test/diff" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/system" + _ "knative.dev/pkg/system/testing" ) +type params struct { + url string + authUsername string + authSecret string + authSecretKey string + authSecretContent string +} + +const sampleTask = `--- +kind: Task +apiVersion: tekton.dev/v1 +metadata: + name: foo +spec: + steps: + - name: step1 + image: scratch` +const emptyStr = "empty" + func TestGetSelector(t *testing.T) { resolver := Resolver{} sel := resolver.GetSelector(context.Background()) @@ -73,7 +104,7 @@ func TestValidateParams(t *testing.T) { resolver := Resolver{} params := map[string]string{} if tc.url != "nourl" { - params[urlParam] = tc.url + params[urlParamType] = tc.url } err := resolver.ValidateParams(contextWithConfig(defaultHttpTimeoutValue), toParams(params)) if tc.expectedErr != nil { @@ -155,7 +186,7 @@ func TestResolve(t *testing.T) { params := []pipelinev1.Param{} if tc.paramSet { params = append(params, pipelinev1.Param{ - Name: urlParam, + Name: urlParamType, Value: *pipelinev1.NewStructuredValues(svr.URL), }) } @@ -211,11 +242,240 @@ func TestResolveNotEnabled(t *testing.T) { } } -func TestInitialize(t *testing.T) { - resolver := Resolver{} - err := resolver.Initialize(context.Background()) - if err != nil { - t.Errorf("unexpected error: %v", err) +func createRequest(params *params) *v1beta1.ResolutionRequest { + rr := &v1beta1.ResolutionRequest{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "resolution.tekton.dev/v1beta1", + Kind: "ResolutionRequest", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "rr", + Namespace: "foo", + CreationTimestamp: metav1.Time{Time: time.Now()}, + Labels: map[string]string{ + resolutioncommon.LabelKeyResolverType: LabelValueHttpResolverType, + }, + }, + Spec: v1beta1.ResolutionRequestSpec{ + Params: []pipelinev1.Param{{ + Name: urlParamType, + Value: *pipelinev1.NewStructuredValues(params.url), + }}, + }, + } + if params.authSecret != "" { + s := params.authSecret + if s == emptyStr { + s = "" + } + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: httpBasicAuthSecret, + Value: *pipelinev1.NewStructuredValues(s), + }) + } + + if params.authUsername != "" { + s := params.authUsername + if s == emptyStr { + s = "" + } + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: httpBasicAuthUsername, + Value: *pipelinev1.NewStructuredValues(s), + }) + } + + if params.authSecretKey != "" { + rr.Spec.Params = append(rr.Spec.Params, pipelinev1.Param{ + Name: httpBasicAuthSecretKey, + Value: *pipelinev1.NewStructuredValues(params.authSecretKey), + }) + } + + return rr +} + +func TestResolverReconcileBasicAuth(t *testing.T) { + var doNotCreate string = "notcreate" + var wrongSecretKey string = "wrongsecretk" + + tests := []struct { + name string + params *params + taskContent string + expectedStatus *v1beta1.ResolutionRequestStatus + expectedErr error + }{ + { + name: "good/URL Resolution", + taskContent: sampleTask, + expectedStatus: internal.CreateResolutionRequestStatusWithData([]byte(sampleTask)), + }, + { + name: "good/URL Resolution with custom basic auth, and custom secret key", + taskContent: sampleTask, + expectedStatus: internal.CreateResolutionRequestStatusWithData([]byte(sampleTask)), + params: ¶ms{ + authSecret: "auth-secret", + authUsername: "auth", + authSecretKey: "token", + authSecretContent: "untoken", + }, + }, + { + name: "good/URL Resolution with custom basic auth no custom secret key", + taskContent: sampleTask, + expectedStatus: internal.CreateResolutionRequestStatusWithData([]byte(sampleTask)), + params: ¶ms{ + authSecret: "auth-secret", + authUsername: "auth", + authSecretContent: "untoken", + }, + }, + { + name: "bad/no url found", + params: ¶ms{}, + expectedErr: errors.New(`invalid resource request "foo/rr": cannot parse url : parse "": empty url`), + }, + { + name: "bad/no secret found", + params: ¶ms{ + authSecret: doNotCreate, + authUsername: "user", + url: "https://blah/blah.com", + }, + expectedErr: errors.New(`error getting "Http" "foo/rr": cannot get API token, secret notcreate not found in namespace foo`), + }, + { + name: "bad/no valid secret key", + params: ¶ms{ + authSecret: "shhhhh", + authUsername: "user", + authSecretKey: wrongSecretKey, + url: "https://blah/blah", + }, + expectedErr: errors.New(`error getting "Http" "foo/rr": cannot get API token, key wrongsecretk not found in secret shhhhh in namespace foo`), + }, + { + name: "bad/missing username params for secret with params", + params: ¶ms{ + authSecret: "shhhhh", + url: "https://blah/blah", + }, + expectedErr: errors.New(`invalid resource request "foo/rr": missing required param http-username when using http-password-secret`), + }, + { + name: "bad/missing password params for secret with username", + params: ¶ms{ + authUsername: "failure", + url: "https://blah/blah", + }, + expectedErr: errors.New(`invalid resource request "foo/rr": missing required param http-password-secret when using http-username`), + }, + { + name: "bad/empty auth username", + params: ¶ms{ + authUsername: emptyStr, + authSecret: "asecret", + url: "https://blah/blah", + }, + expectedErr: errors.New(`invalid resource request "foo/rr": value http-username cannot be empty`), + }, + { + name: "bad/empty auth password", + params: ¶ms{ + authUsername: "auser", + authSecret: emptyStr, + url: "https://blah/blah", + }, + expectedErr: errors.New(`invalid resource request "foo/rr": value http-password-secret cannot be empty`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolver := &Resolver{} + ctx, _ := ttesting.SetupFakeContext(t) + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, tt.taskContent) + })) + p := tt.params + if p == nil { + p = ¶ms{} + } + if p.url == "" && tt.taskContent != "" { + p.url = svr.URL + } + request := createRequest(p) + cfg := make(map[string]string) + d := test.Data{ + ConfigMaps: []*corev1.ConfigMap{{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: resolverconfig.ResolversNamespace(system.Namespace()), + }, + Data: cfg, + }, { + ObjectMeta: metav1.ObjectMeta{ + Namespace: resolverconfig.ResolversNamespace(system.Namespace()), + Name: resolverconfig.GetFeatureFlagsConfigName(), + }, + Data: map[string]string{ + "enable-http-resolver": "true", + }, + }}, + ResolutionRequests: []*v1beta1.ResolutionRequest{request}, + } + var expectedStatus *v1beta1.ResolutionRequestStatus + if tt.expectedStatus != nil { + expectedStatus = tt.expectedStatus.DeepCopy() + if tt.expectedErr == nil { + if tt.taskContent != "" { + h := sha256.New() + h.Write([]byte(tt.taskContent)) + sha256CheckSum := hex.EncodeToString(h.Sum(nil)) + refsrc := &pipelinev1.RefSource{ + URI: svr.URL, + Digest: map[string]string{ + "sha256": sha256CheckSum, + }, + } + expectedStatus.RefSource = refsrc + expectedStatus.Source = refsrc + } + } else { + expectedStatus.Status.Conditions[0].Message = tt.expectedErr.Error() + } + } + frtesting.RunResolverReconcileTest(ctx, t, d, resolver, request, expectedStatus, tt.expectedErr, func(resolver framework.Resolver, testAssets test.Assets) { + if err := resolver.Initialize(ctx); err != nil { + t.Errorf("unexpected error: %v", err) + } + if tt.params == nil { + return + } + if tt.params.authSecret != "" && tt.params.authSecret != doNotCreate { + secretKey := tt.params.authSecretKey + if secretKey == wrongSecretKey { + secretKey = "randomNotOund" + } + if secretKey == "" { + secretKey = defaultBasicAuthSecretKey + } + tokenSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: tt.params.authSecret, + Namespace: request.GetNamespace(), + }, + Data: map[string][]byte{ + secretKey: []byte(base64.StdEncoding.Strict().EncodeToString([]byte(tt.params.authSecretContent))), + }, + } + if _, err := testAssets.Clients.Kube.CoreV1().Secrets(request.GetNamespace()).Create(ctx, tokenSecret, metav1.CreateOptions{}); err != nil { + t.Fatalf("failed to create test token secret: %v", err) + } + } + }) + }) } }