From 08fecbdc074050fb085a1f0551dfc7888bafc0c6 Mon Sep 17 00:00:00 2001 From: Shivam Mukhade Date: Tue, 11 Oct 2022 12:09:39 +0530 Subject: [PATCH] Auto configuring newly created GitHub Repos with GH App supports template for generating namespace for repos Signed-off-by: Shivam Mukhade --- config/201-controller-role.yaml | 5 +- config/302-pac-configmap.yaml | 10 ++ docs/content/docs/install/settings.md | 24 +++- pkg/adapter/adapter.go | 21 ++- pkg/adapter/adapter_test.go | 4 + pkg/params/info/pac.go | 4 + pkg/params/run.go | 8 ++ pkg/provider/github/github.go | 11 +- pkg/provider/github/repository.go | 112 +++++++++++++++ pkg/provider/github/repository_test.go | 190 +++++++++++++++++++++++++ 10 files changed, 380 insertions(+), 9 deletions(-) create mode 100644 pkg/provider/github/repository.go create mode 100644 pkg/provider/github/repository_test.go diff --git a/config/201-controller-role.yaml b/config/201-controller-role.yaml index c57f37090..81c5d8d7e 100644 --- a/config/201-controller-role.yaml +++ b/config/201-controller-role.yaml @@ -62,12 +62,15 @@ metadata: app.kubernetes.io/instance: default app.kubernetes.io/part-of: pipelines-as-code rules: + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["create"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "create"] - apiGroups: ["pipelinesascode.tekton.dev"] resources: ["repositories"] - verbs: ["list"] + verbs: ["create", "list"] - apiGroups: ["tekton.dev"] resources: ["pipelineruns"] verbs: ["get", "create", "patch"] diff --git a/config/302-pac-configmap.yaml b/config/302-pac-configmap.yaml index 5c733b158..c40762bad 100644 --- a/config/302-pac-configmap.yaml +++ b/config/302-pac-configmap.yaml @@ -50,6 +50,16 @@ data: # if defined then applies to all pipelineRun who doesn't have max-keep-runs annotation default-max-keep-runs: "" + # Whether to auto configure newly created repositories, this will create a new namespace + # and repository CR, supported only with GitHub App + auto-configure-new-github-repo: "false" + + # add a template to generate name for namespace for your auto configured github repo + # supported fields are repo_owner, repo_name + # eg. if defined as `{{repo_owner}}-{{repo_name}}-ci`, then namespace generated for repository + # https://github.com/owner/repo will be `owner-repo-ci` + auto-configure-repo-namespace-template: "" + kind: ConfigMap metadata: name: pipelines-as-code diff --git a/docs/content/docs/install/settings.md b/docs/content/docs/install/settings.md index 5851acc02..fa8c7f27e 100644 --- a/docs/content/docs/install/settings.md +++ b/docs/content/docs/install/settings.md @@ -60,9 +60,31 @@ There is a few things you can configure through the configmap This allows user to define a default limit for max-keep-run value. If defined then it's applied to all the pipelineRun which do not have `max-keep-runs` annotation. + +* `auto-configure-new-github-repo` + + Let you autoconfigure newly created GitHub repositories. On creation of a new repository, Pipelines As Code will set up a namespace + for your repository and create a Repository CR. + + This feature is disabled by default and is only supported with GitHub App. -## Pipelines-As-Code Info + ****NOTE****: If you have a GitHub App already setup then verify if `Repository` event is subscribed. + +* `auto-configure-repo-namespace-template` + + If `auto-configure-new-github-repo` is enabled then you can provide a template for generating the namespace for your new repository. + By default, the namespace will be generated using this format `{{repo_name}}-pipelines`. + You can override the default using the following variables + + * `{{repo_owner}}`: The repository owner. + * `{{repo_name}}`: The repository name. + + for example. if the template is defined as `{{repo_owner}}-{{repo_name}}-ci`, then the namespace generated for repository + `https://github.com/owner/repo` will be `owner-repo-ci` + +## Pipelines-As-Code Info + There are a settings exposed through a configmap which any authenticated user can access to know about Pipeline as Code. diff --git a/pkg/adapter/adapter.go b/pkg/adapter/adapter.go index 06b3836a1..44db56001 100644 --- a/pkg/adapter/adapter.go +++ b/pkg/adapter/adapter.go @@ -106,8 +106,6 @@ func (l listener) handleEvent() http.HandlerFunc { return } - // read request Body - // event body payload, err := io.ReadAll(request.Body) if err != nil { @@ -128,6 +126,25 @@ func (l listener) handleEvent() http.HandlerFunc { var logger *zap.SugaredLogger l.event = info.NewEvent() + + // if repository auto configuration is enabled then check if its a valid event + if l.run.Info.Pac.AutoConfigureNewRepo { + detected, configuring, err := github.ConfigureRepository(ctx, l.run, request, string(payload), l.logger) + if detected { + if configuring && err == nil { + l.writeResponse(response, http.StatusCreated, "configured") + return + } + if configuring && err != nil { + l.logger.Errorf("repository auto-configure has failed, err: %v", err) + l.writeResponse(response, http.StatusOK, "failed to configure") + return + } + l.writeResponse(response, http.StatusOK, "skipped event") + return + } + } + isIncoming, targettedRepo, err := l.detectIncoming(ctx, request, payload) if err != nil { l.logger.Errorf("error processing incoming webhook: %v", err) diff --git a/pkg/adapter/adapter_test.go b/pkg/adapter/adapter_test.go index 6ba631b40..1b9069883 100644 --- a/pkg/adapter/adapter_test.go +++ b/pkg/adapter/adapter_test.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-github/v47/github" "github.com/openshift-pipelines/pipelines-as-code/pkg/params" "github.com/openshift-pipelines/pipelines-as-code/pkg/params/clients" + "github.com/openshift-pipelines/pipelines-as-code/pkg/params/info" testclient "github.com/openshift-pipelines/pipelines-as-code/pkg/test/clients" "go.uber.org/zap" zapobserver "go.uber.org/zap/zaptest/observer" @@ -33,6 +34,9 @@ func TestHandleEvent(t *testing.T) { Clients: clients.Clients{ PipelineAsCode: cs.PipelineAsCode, }, + Info: info.Info{ + Pac: &info.PacOpts{AutoConfigureNewRepo: false}, + }, }, logger: getLogger(), } diff --git a/pkg/params/info/pac.go b/pkg/params/info/pac.go index 38160c19e..16f9d6067 100644 --- a/pkg/params/info/pac.go +++ b/pkg/params/info/pac.go @@ -23,6 +23,10 @@ type PacOpts struct { // bitbucket cloud specific fields BitbucketCloudCheckSourceIP bool BitbucketCloudAdditionalSourceIP string + + // GitHub App specific + AutoConfigureNewRepo bool + AutoConfigureRepoNamespaceTemplate string } func (p *PacOpts) AddFlags(cmd *cobra.Command) error { diff --git a/pkg/params/run.go b/pkg/params/run.go index 85341ad29..8f450b249 100644 --- a/pkg/params/run.go +++ b/pkg/params/run.go @@ -140,6 +140,14 @@ func (r *Run) UpdatePACInfo(ctx context.Context) error { } } + if autoConfigure, ok := cfg.Data["auto-configure-new-github-repo"]; ok { + r.Info.Pac.AutoConfigureNewRepo = StringToBool(autoConfigure) + } + + if nsTemplate, ok := cfg.Data["auto-configure-repo-namespace-template"]; ok { + r.Info.Pac.AutoConfigureRepoNamespaceTemplate = nsTemplate + } + return nil } diff --git a/pkg/provider/github/github.go b/pkg/provider/github/github.go index a6bbeee1c..4199c1c27 100644 --- a/pkg/provider/github/github.go +++ b/pkg/provider/github/github.go @@ -29,11 +29,12 @@ const ( ) type Provider struct { - Client *github.Client - Logger *zap.SugaredLogger - Token, APIURL *string - ApplicationID *int64 - providerName string + Client *github.Client + Logger *zap.SugaredLogger + Token, APIURL *string + ApplicationID *int64 + providerName string + AutoConfigureNewRepos bool } // splitGithubURL Take a Github url and split it with org/repo path ref, supports rawURL diff --git a/pkg/provider/github/repository.go b/pkg/provider/github/repository.go new file mode 100644 index 000000000..69dd18ae0 --- /dev/null +++ b/pkg/provider/github/repository.go @@ -0,0 +1,112 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "github.com/google/go-github/v47/github" + "github.com/openshift-pipelines/pipelines-as-code/pkg/apis/pipelinesascode/v1alpha1" + "github.com/openshift-pipelines/pipelines-as-code/pkg/formatting" + "github.com/openshift-pipelines/pipelines-as-code/pkg/params" + "github.com/openshift-pipelines/pipelines-as-code/pkg/params/clients" + "github.com/openshift-pipelines/pipelines-as-code/pkg/templates" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const defaultNsTemplate = "%v-pipelines" + +func ConfigureRepository(ctx context.Context, run *params.Run, req *http.Request, payload string, logger *zap.SugaredLogger) (bool, bool, error) { + // gitea set x-github-event too, so skip it for the gitea driver + if h := req.Header.Get("X-Gitea-Event-Type"); h != "" { + return false, false, nil + } + event := req.Header.Get("X-Github-Event") + if event != "repository" { + return false, false, nil + } + + eventInt, err := github.ParseWebHook(event, []byte(payload)) + if err != nil { + return true, false, err + } + _ = json.Unmarshal([]byte(payload), &eventInt) + repoEvent, _ := eventInt.(*github.RepositoryEvent) + + if repoEvent.GetAction() != "created" { + logger.Infof("github: repository event \"%v\" is not supported", repoEvent.GetAction()) + return true, false, nil + } + + logger.Infof("github: configuring repository cr for repo: %v", repoEvent.Repo.GetHTMLURL()) + if err := createRepository(ctx, run.Info.Pac.AutoConfigureRepoNamespaceTemplate, run.Clients, repoEvent, logger); err != nil { + logger.Errorf("failed repository creation: %v", err) + return true, true, err + } + + return true, true, nil +} + +func createRepository(ctx context.Context, nsTemplate string, clients clients.Clients, gitEvent *github.RepositoryEvent, logger *zap.SugaredLogger) error { + repoNsName, err := generateNamespaceName(nsTemplate, gitEvent) + if err != nil { + return fmt.Errorf("failed to generate namespace for repo: %w", err) + } + + logger.Info("github: generated namespace name: ", repoNsName) + + // create namespace + repoNs := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: repoNsName, + }, + } + repoNs, err = clients.Kube.CoreV1().Namespaces().Create(ctx, repoNs, metav1.CreateOptions{}) + if err != nil && !errors.IsAlreadyExists(err) { + return fmt.Errorf("failed to create namespace %v: %w", repoNs.Name, err) + } + + if errors.IsAlreadyExists(err) { + logger.Infof("github: namespace %v already exists, creating repository", repoNsName) + } else { + logger.Info("github: created repository namespace: ", repoNs.Name) + } + + // create repository + repo := &v1alpha1.Repository{ + ObjectMeta: metav1.ObjectMeta{ + Name: repoNsName, + Namespace: repoNsName, + }, + Spec: v1alpha1.RepositorySpec{ + URL: gitEvent.Repo.GetHTMLURL(), + }, + } + repo, err = clients.PipelineAsCode.PipelinesascodeV1alpha1().Repositories(repoNsName).Create(ctx, repo, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create repository for repo: %v: %w", gitEvent.Repo.GetHTMLURL(), err) + } + logger.Infof("github: repository created: %s/%s ", repo.Namespace, repo.Name) + return nil +} + +func generateNamespaceName(nsTemplate string, gitEvent *github.RepositoryEvent) (string, error) { + repoOwner, repoName, err := formatting.GetRepoOwnerSplitted(gitEvent.Repo.GetHTMLURL()) + if err != nil { + return "", fmt.Errorf("failed to parse git repo url: %w", err) + } + + if nsTemplate == "" { + return fmt.Sprintf(defaultNsTemplate, repoName), nil + } + + maptemplate := map[string]string{ + "repo_owner": repoOwner, + "repo_name": repoName, + } + return templates.ReplacePlaceHoldersVariables(nsTemplate, maptemplate), nil +} diff --git a/pkg/provider/github/repository_test.go b/pkg/provider/github/repository_test.go new file mode 100644 index 000000000..f98bde919 --- /dev/null +++ b/pkg/provider/github/repository_test.go @@ -0,0 +1,190 @@ +package github + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "testing" + + "github.com/google/go-github/v47/github" + "github.com/openshift-pipelines/pipelines-as-code/pkg/params" + "github.com/openshift-pipelines/pipelines-as-code/pkg/params/clients" + "github.com/openshift-pipelines/pipelines-as-code/pkg/params/info" + testclient "github.com/openshift-pipelines/pipelines-as-code/pkg/test/clients" + "go.uber.org/zap" + zapobserver "go.uber.org/zap/zaptest/observer" + "gotest.tools/v3/assert" + v12 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + rtesting "knative.dev/pkg/reconciler/testing" +) + +func TestConfigureRepository(t *testing.T) { + observer, _ := zapobserver.New(zap.InfoLevel) + logger := zap.New(observer).Sugar() + + testEvent := github.RepositoryEvent{Action: github.String("updated")} + repoUpdatedEvent, err := json.Marshal(testEvent) + assert.NilError(t, err) + + testRepoName := "test-repo" + testRepoOwner := "pac" + testURL := fmt.Sprintf("https://github.com/%v/%v", testRepoOwner, testRepoName) + + testCreateEvent := github.RepositoryEvent{Action: github.String("created"), Repo: &github.Repository{HTMLURL: github.String(testURL)}} + repoCreateEvent, err := json.Marshal(testCreateEvent) + assert.NilError(t, err) + + tests := []struct { + name string + request *http.Request + eventType string + event []byte + detected bool + configuring bool + wantErr string + expectedNs string + nsTemplate string + testData testclient.Data + }{ + { + name: "non supported event", + event: []byte{}, + eventType: "push", + detected: false, + configuring: false, + wantErr: "", + testData: testclient.Data{}, + }, + { + name: "repo updated event", + event: repoUpdatedEvent, + eventType: "repository", + detected: true, + configuring: false, + wantErr: "", + expectedNs: "", + testData: testclient.Data{}, + }, + { + name: "repo create event with no ns template", + event: repoCreateEvent, + eventType: "repository", + detected: true, + configuring: true, + wantErr: "", + expectedNs: "test-repo-pipelines", + testData: testclient.Data{}, + }, + { + name: "repo create event with ns template", + event: repoCreateEvent, + eventType: "repository", + detected: true, + configuring: true, + wantErr: "", + expectedNs: "pac-test-repo-ci", + nsTemplate: "{{repo_owner}}-{{repo_name}}-ci", + testData: testclient.Data{}, + }, + { + name: "repo create event with ns already exist", + event: repoCreateEvent, + eventType: "repository", + detected: true, + configuring: true, + wantErr: "", + expectedNs: "test-repo-pipelines", + testData: testclient.Data{ + Namespaces: []*v12.Namespace{ + { + ObjectMeta: v1.ObjectMeta{ + Name: "test-repo-pipelines", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, _ := rtesting.SetupFakeContext(t) + cs, _ := testclient.SeedTestData(t, ctx, tt.testData) + run := ¶ms.Run{ + Clients: clients.Clients{ + PipelineAsCode: cs.PipelineAsCode, + Kube: cs.Kube, + }, + Info: info.Info{ + Pac: &info.PacOpts{ + AutoConfigureRepoNamespaceTemplate: tt.nsTemplate, + }, + }, + } + req, err := http.NewRequestWithContext(context.TODO(), "POST", "URL", bytes.NewReader(tt.event)) + if err != nil { + t.Fatalf("error creating request: %s", err) + } + req.Header.Set("X-Github-Event", tt.eventType) + + detected, configuring, err := ConfigureRepository(ctx, run, req, string(tt.event), logger) + assert.Equal(t, detected, tt.detected) + assert.Equal(t, configuring, tt.configuring) + + if tt.wantErr != "" { + assert.Equal(t, tt.wantErr, err.Error()) + } else { + assert.NilError(t, err) + } + + if tt.configuring { + ns, err := run.Clients.Kube.CoreV1().Namespaces().Get(ctx, tt.expectedNs, v1.GetOptions{}) + assert.NilError(t, err) + assert.Equal(t, ns.Name, tt.expectedNs) + + repo, err := run.Clients.PipelineAsCode.PipelinesascodeV1alpha1().Repositories(tt.expectedNs).Get(ctx, tt.expectedNs, v1.GetOptions{}) + assert.NilError(t, err) + assert.Equal(t, repo.Name, tt.expectedNs) + } + }) + } +} + +func TestGetNamespace(t *testing.T) { + tests := []struct { + name string + nsTemplate string + gitEvent *github.RepositoryEvent + want string + }{ + { + name: "no template", + nsTemplate: "", + gitEvent: &github.RepositoryEvent{ + Repo: &github.Repository{ + HTMLURL: github.String("https://github.com/user/pac"), + }, + }, + want: "pac-pipelines", + }, + { + name: "template", + nsTemplate: "{{repo_owner}}-{{repo_name}}-ci", + gitEvent: &github.RepositoryEvent{ + Repo: &github.Repository{ + HTMLURL: github.String("https://github.com/user/pac"), + }, + }, + want: "user-pac-ci", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := generateNamespaceName(tt.nsTemplate, tt.gitEvent) + assert.NilError(t, err) + assert.Equal(t, got, tt.want) + }) + } +}