From aaa5c0ce15df6c41e71c45ac55dd772fe2ccfe41 Mon Sep 17 00:00:00 2001 From: Tze Yang Ng Date: Sat, 2 Mar 2024 10:46:00 +0800 Subject: [PATCH] fix --- client/client_test.go | 93 -------------- client/dockerhub_api_client.go | 49 +++---- client/setup.go | 227 --------------------------------- main_test.go | 187 --------------------------- 4 files changed, 26 insertions(+), 530 deletions(-) delete mode 100644 client/client_test.go delete mode 100644 client/setup.go delete mode 100644 main_test.go diff --git a/client/client_test.go b/client/client_test.go deleted file mode 100644 index 84a85aa..0000000 --- a/client/client_test.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build integration - -package client - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -/* -Test ShouldDeploy for a multitude of conditions. -*/ -func TestAutoDeployLatestTag(t *testing.T) { - te := SetUpClientTest(t) - defer te.TearDown() - - // populate with new tags - tags := []string{"test", "v0.1.0"} - for _, tag := range tags { - te.PushNewTag(tag, "latest") - } - - // update the cache with digests - te.Clients.PopulateCaches(te.TestRepoName) - - /* - resolves correctly for autodeployment of latest - */ - - // push the new tag - newTag := "v0.0.2" - te.PushNewTag(newTag, "latest") - - shouldDeploy, _ := te.Clients.ShouldDeploy(te.TestRepoName) - tagToDeploy, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName) - // v0.0.2 is not a new release version - assert.False(t, shouldDeploy) - assert.Equal(t, tagToDeploy, "v0.1.0") - - // push the new tag - newTag = "v0.2.0" - te.PushNewTag(newTag, "latest") - - shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) - // v0.2.0 is a new release version, - assert.True(t, shouldDeploy) - assert.Equal(t, tagToDeploy, newTag) - - // push the new tag - newTag = "v0.0.9" - te.PushNewTag(newTag, "latest") - - shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) - // v0.0.9 is not a new release version - assert.False(t, shouldDeploy) - assert.Equal(t, tagToDeploy, "v0.2.0") - - /* - resolves correctly for autodeployment of custom tags - */ - - // set to "test:latest" - newTag = "test" - te.UpdatePinnedTag(newTag) - - shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) - // should not deploy as all tags are based on the digest "latest", even though - // the pinned tag is changed - assert.False(t, shouldDeploy) - assert.Equal(t, "test", tagToDeploy) - - // "test" is now based on "alpine", rather than the original "latest" - te.PushNewTag(newTag, "alpine") - - shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - tagToDeploy, _ = te.Clients.GetFormattedPinnedTag(te.TestRepoName) - assert.True(t, shouldDeploy) - assert.Equal(t, "test", tagToDeploy) - - // "test" is back to "latest", but autoDeploy is off - _ = te.Clients.PostgresClient.UpdateAutoDeployFlag(te.TestRepoName, false) - te.PushNewTag(newTag, "alpine") - - shouldDeploy, _ = te.Clients.ShouldDeploy(te.TestRepoName) - autoDeploy, _ := te.Clients.PostgresClient.GetAutoDeployFlag(te.TestRepoName) - assert.False(t, autoDeploy) - assert.False(t, shouldDeploy) - assert.Equal(t, "test", tagToDeploy) -} diff --git a/client/dockerhub_api_client.go b/client/dockerhub_api_client.go index 41bd7f0..e40cd91 100644 --- a/client/dockerhub_api_client.go +++ b/client/dockerhub_api_client.go @@ -24,13 +24,19 @@ type AuthenticateResp struct { Token string `json:"token"` } -type CheckImageResp struct { - Results []TagWithStatus `json:"results"` +type GetRepoTagsResp struct { + Results []RepoTag `json:"results"` } -type TagWithStatus struct { - Tag string `json:"tag"` - IsCurrent bool `json:"is_current"` +type RepoTag struct { + // Tag string `json:"tag"` + // IsCurrent bool `json:"is_current"` + Name string `json:"name"` + Images []RepoImage `json:"images"` +} + +type RepoImage struct { + Digest string `json:"digest"` } type GetTagDigestResp struct { @@ -38,8 +44,8 @@ type GetTagDigestResp struct { } type GetTagDigestResult struct { - Digest string `json:"digest"` - Tags []TagWithStatus `json:"tags"` + Digest string `json:"digest"` + Tags []RepoTag `json:"tags"` } func InitializeDockerhubApi(conf *viper.Viper) (*DockerhubApi, error) { @@ -89,10 +95,9 @@ func (api *DockerhubApi) Authenticate() (*string, error) { return &deserialized.Token, nil } -func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag string) ( +func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, tagName string) ( *bool, error) { - endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images/%s/tags", - api.namespace, repository, digest) + endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/tags", api.namespace, repository) addr := fmt.Sprintf("%s%s", api.url, endpoint) log.LogAppInfo(fmt.Sprintf("dockerhub check if image is current, url=%s", addr)) @@ -129,28 +134,28 @@ func (api *DockerhubApi) CheckImageIsCurrent(repository, digest string, checkTag return nil, errors.New(errMsg) } - var deserialized CheckImageResp + var deserialized GetRepoTagsResp json.Unmarshal(body, &deserialized) for _, item := range deserialized.Results { - if item.Tag == checkTag { - return &item.IsCurrent, nil + if item.Name == tagName { + isCurrent := item.Images[0].Digest == digest + return &isCurrent, nil } } // image tag does not match the tag to be checked // this means the cached digest belongs to a previous tag // return false (not current) because we want the cache to be updated - log.LogAppInfo(fmt.Sprintf(fmt.Sprintf("Digest %s does not have tag %s", digest, checkTag))) + log.LogAppInfo(fmt.Sprintf(fmt.Sprintf("Digest %s does not have tag %s", digest, tagName))) isCurrent := false return &isCurrent, nil } // Note that the digest returned from the Dockerhub API does not match // the digest from docker registry manifest V2 API -func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string) ( +func (api *DockerhubApi) GetTagDigestFromApi(repository string, tagName string) ( *string, error) { - endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/images?", api.namespace, - repository) + endpoint := fmt.Sprintf("/v2/namespaces/%s/repositories/%s/tags", api.namespace, repository) queryParams := fmt.Sprintf("currently_tagged=%s&page_size=%s&ordering=%s", "true", "100", "-last_activity") @@ -190,17 +195,15 @@ func (api *DockerhubApi) GetTagDigestFromApi(repository string, checkTag string) return nil, errors.New(errMsg) } - var deserialized GetTagDigestResp + var deserialized GetRepoTagsResp json.Unmarshal(body, &deserialized) for _, item := range deserialized.Results { - for _, tag := range item.Tags { - if tag.Tag == checkTag && tag.IsCurrent { - return &item.Digest, nil - } + if item.Name == tagName { + return &item.Images[0].Digest, nil } } // image tag not found in repository's 100 last active tags - return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", checkTag, repository)) + return nil, errors.New(fmt.Sprintf("Tag %s not found in repository %s", tagName, repository)) } diff --git a/client/setup.go b/client/setup.go deleted file mode 100644 index f442287..0000000 --- a/client/setup.go +++ /dev/null @@ -1,227 +0,0 @@ -package client - -import ( - "fmt" - "testing" - - "github.com/dsaidgovsg/registrywatcher/config" - "github.com/dsaidgovsg/registrywatcher/log" - "github.com/dsaidgovsg/registrywatcher/testutils" - "github.com/dsaidgovsg/registrywatcher/utils" - nomad "github.com/hashicorp/nomad/api" - "github.com/spf13/viper" - - "encoding/json" - "net/http" - "net/http/httptest" - - "github.com/gorilla/mux" -) - -type testEngine struct { - containerIDs []string // keep track of container IDs to be destroyed at tearDown() - Conf *viper.Viper - helper *testutils.TestHelper - Clients *Clients - ImageTagMap map[string][]TagWithStatus - Ts *httptest.Server - TestRepoName string -} - -func (te *testEngine) printState() { - registryTags, _ := te.Clients.DockerRegistryClient.GetAllTags(te.TestRepoName) - fmt.Println("registry tags", registryTags) - - cachedTags, _ := te.Clients.GetCachedTags(te.TestRepoName) - fmt.Println("cached tags", cachedTags) - - pinnedTag, _ := te.Clients.GetFormattedPinnedTag(te.TestRepoName) - tagDigest, _ := te.Clients.DockerhubApi.GetTagDigestFromApi(te.TestRepoName, pinnedTag) - fmt.Println("new tag digest", *tagDigest) - - cachedTagDigest, _ := te.Clients.GetCachedTagDigest(te.TestRepoName) - fmt.Println("cached tag digest", cachedTagDigest) -} - -func SetUpClientTest(t *testing.T) *testEngine { - conf := config.SetUpConfig("test") - - te := testEngine{ - Conf: conf, - helper: testutils.NewTestHelper(conf), - } - - // start registry - regID, _, err := te.helper.StartRegistry() - if err != nil { - te.helper.RemoveContainer(regID) - panic(fmt.Errorf("starting registry container failed: %v", err)) - } - - // start postgres - pgID, err := te.helper.StartPostgres() - if err != nil { - te.helper.RemoveContainer(pgID, regID) - panic(fmt.Errorf("starting postgres container failed: %v", err)) - } - - // add registry and postgres container ID to be removed later - te.containerIDs = append(te.containerIDs, regID) - te.containerIDs = append(te.containerIDs, pgID) - - // initialize the clients - te.Clients = SetUpTestClients(t, conf) - - // we use this so much might as well keep it in the struct - te.TestRepoName = te.Conf.GetStringSlice("watched_repositories")[0] - - // initialize mock imageTag store - te.ImageTagMap = make(map[string][]TagWithStatus) - - // initialize mock Dockerhub server - router := mux.NewRouter() - - router.HandleFunc("/v2/users/login", func(res http.ResponseWriter, req *http.Request) { - res.Write([]byte(`{"token": "test token"}`)) - }).Methods("GET") - - router.HandleFunc("/v2/namespaces/{namespace}/repositories/{repo}/images/{digest}/tags", - func(res http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - digest := vars["digest"] - - res.Header().Set("Content-Type", "application/json") - res.WriteHeader(http.StatusOK) - - if tags, ok := te.ImageTagMap[digest]; ok { - results := CheckImageResp{Results: tags} - data, _ := json.Marshal(results) - res.Write(data) - } else { - results := CheckImageResp{Results: nil} - data, _ := json.Marshal(results) - res.Write(data) - } - }).Methods("GET") - - router.HandleFunc("/v2/namespaces/namespace/repositories/testrepo/images", - func(res http.ResponseWriter, req *http.Request) { - var resSlice []GetTagDigestResult - for image, tags := range te.ImageTagMap { - resSlice = append(resSlice, GetTagDigestResult{Digest: image, Tags: tags}) - } - res.Header().Set("Content-Type", "application/json") - res.WriteHeader(http.StatusOK) - results := GetTagDigestResp{Results: resSlice} - data, _ := json.Marshal(results) - res.Write(data) - }) - - ts := httptest.NewServer(router) - te.Ts = ts - - dockerhubApi := DockerhubApi{ - url: ts.URL, - namespace: "namespace", - username: "username", - secret: "secret", - token: "fake token", - } - - te.Clients.DockerhubApi = &dockerhubApi - return &te -} - -func (te *testEngine) RegisterJob() { - jobID := utils.GetRepoNomadJob(te.Conf, te.TestRepoName) - tags, _ := te.Clients.DockerRegistryClient.GetAllTags(te.TestRepoName) - dockerImage := fmt.Sprintf("%s:%s", te.TestRepoName, tags[0]) - job := testJob(jobID, dockerImage) - jobs := te.Clients.NomadClient.nc.Jobs() - _, _, err := jobs.Register(job, nil) - if err != nil { - panic(fmt.Errorf("starting nomad job failed: %v", err)) - } -} - -func testJob(jobID, dockerImage string) *nomad.Job { - count := 1 - name := "job" - taskName := "test" - jobType := "service" - region := "region1" - return &nomad.Job{ - ID: &jobID, - Name: &name, - Type: &jobType, - Datacenters: []string{"dc-1"}, - Region: ®ion, - TaskGroups: []*nomad.TaskGroup{ - { - Name: &taskName, - Count: &count, - Tasks: []*nomad.Task{ - { - Name: "test", - Driver: "docker", - Config: map[string]interface{}{ - "image": dockerImage, - }, - }, - }, - }, - }, - } - -} - -func (te *testEngine) TearDown() { - te.Clients.NomadServer.Stop() - for _, containerID := range te.containerIDs { - if err := te.helper.RemoveContainer(containerID); err != nil { - log.LogAppErr("Couldn't remove container", err) - } - } - te.Ts.Close() -} - -/* -For tests, note that the base image tags "latest" and "alpine" are taken to be equivalent -to image digest strings in the mock image-tag store. The mock image tag store is used -because we are mocking the Dockerhub API server. While in the real world, image digests -are SHA hashes, this still allows us to test the auto-deploy behaviour as we use the -digest string to check if the underlying image is changed. -*/ -func (te *testEngine) PushNewTag(namedTag, actualTag string) { - _, registryDomain, registryPrefix, _ := utils.ExtractRegistryInfo(te.Conf, te.TestRepoName) - mockImageName := utils.ConstructImageName(registryDomain, registryPrefix, te.TestRepoName, namedTag) - publicImageName := fmt.Sprintf("%s:%s", te.Conf.GetString("base_public_image"), actualTag) - err := te.helper.AddImageToRegistry(publicImageName, mockImageName) - - imageDigest := actualTag - if _, ok := te.ImageTagMap[imageDigest]; ok { - te.ImageTagMap[imageDigest] = append(te.ImageTagMap[imageDigest], TagWithStatus{namedTag, true}) - } else { - te.ImageTagMap[imageDigest] = []TagWithStatus{{namedTag, true}} - } - - // make is_current false for older images holding the same tag - for image, tags := range te.ImageTagMap { - for i, tag := range tags { - if tag.Tag == namedTag && image != imageDigest { - tags[i].IsCurrent = false - } - } - } - - if err != nil { - panic(fmt.Errorf("couldn't add image to registry: %v", err)) - } -} - -func (te *testEngine) UpdatePinnedTag(newTag string) { - err := te.Clients.PostgresClient.UpdatePinnedTag(te.TestRepoName, newTag) - if err != nil { - panic(fmt.Errorf("couldn't update postgres client pinned_tag: %v", err)) - } -} diff --git a/main_test.go b/main_test.go deleted file mode 100644 index 659589b..0000000 --- a/main_test.go +++ /dev/null @@ -1,187 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "testing" - - "github.com/dsaidgovsg/registrywatcher/client" - "github.com/dsaidgovsg/registrywatcher/utils" - "github.com/stretchr/testify/assert" -) - -type RepoSummaryResult struct { - Map map[string]string `json:"testrepo" binding:"required"` -} - -func TestRepoSummaryHandler(t *testing.T) { - te := client.SetUpClientTest(t) - router := SetUpRouter(te.Conf, te.Clients) - defer te.TearDown() - var rtn RepoSummaryResult - - // populate with new tags - tags := []string{"v1.0.0"} - for _, tag := range tags { - te.PushNewTag(tag, "latest") - } - te.Clients.PopulateCaches(te.TestRepoName) - - request, _ := http.NewRequest("GET", "/repos", nil) - response := httptest.NewRecorder() - router.ServeHTTP(response, request) - _ = json.NewDecoder(response.Body).Decode(&rtn) - - // tag is latest by default, which is v1.0.0 - tag := rtn.Map["pinned_tag_value"] - assert.Equal(t, "v1.0.0", tag, "OK tags correspond") - - // update pinnedTag - newTag := "v0.1.0" - te.UpdatePinnedTag(newTag) - - request, _ = http.NewRequest("GET", "/repos", nil) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - _ = json.NewDecoder(response.Body).Decode(&rtn) - - // after setting tag to newTag, endpoint should reflect the updated status - tag = rtn.Map["pinned_tag_value"] - assert.Equal(t, newTag, tag, "OK tags correspond") -} - -type GetTagResult struct { - Tag string `json:"repo_tag" binding:"required"` -} - -// also tests RepinnedTagHandler since setUp and tearDown is expensive -func TestGetTagHandler(t *testing.T) { - te := client.SetUpClientTest(t) - router := SetUpRouter(te.Conf, te.Clients) - defer te.TearDown() - var rtn GetTagResult - - // populate with new tags - tags := []string{"v1.0.0"} - for _, tag := range tags { - te.PushNewTag(tag, "latest") - } - - request, _ := http.NewRequest("GET", fmt.Sprintf("/tags/%s", te.TestRepoName), nil) - response := httptest.NewRecorder() - router.ServeHTTP(response, request) - _ = json.NewDecoder(response.Body).Decode(&rtn) - - // tag is latest by default, which is v1.0.0 - tag := rtn.Tag - assert.Equal(t, "v1.0.0", tag, "OK tags correspond") - - // update pinnedTag - newTag := "v0.0.1" - te.UpdatePinnedTag(newTag) - - request, _ = http.NewRequest("GET", fmt.Sprintf("/tags/%s", te.TestRepoName), nil) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - _ = json.NewDecoder(response.Body).Decode(&rtn) - - // tag is latest by default, which is v1.0.0 - tag = rtn.Tag - assert.Equal(t, newTag, tag, "OK tags correspond") - - // push new tag - newTag = "v2.0.0" - te.PushNewTag(newTag, "latest") - - // reset pinnedTag - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s/reset", te.TestRepoName), nil) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - - // tag is reset to latest, which is now v2.0.0 - tag, _ = te.Clients.PostgresClient.GetPinnedTag(te.TestRepoName) - assert.Equal(t, tag, "", "OK tags is latest") - tags, _ = te.Clients.DockerRegistryClient.GetAllTags(te.TestRepoName) - tagValue, _ := utils.GetLatestReleaseTag(tags) - assert.Equal(t, newTag, tagValue, "OK latest tag is v2.0.0") - - // test querying on a repo that's not being watched - request, _ = http.NewRequest("GET", "/tags/nonexistent-repo", nil) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - _ = json.NewDecoder(response.Body).Decode(&rtn) - assert.Equal(t, 400, response.Code, "OK response is expected") -} - -func TestDeployTagHandler(t *testing.T) { - te := client.SetUpClientTest(t) - router := SetUpRouter(te.Conf, te.Clients) - defer te.TearDown() - - // populate with new tags - tags := []string{"v1.0.0"} - for _, tag := range tags { - te.PushNewTag(tag, "latest") - } - - // test with missing params (at least 1 argument must be provided) - data := []byte(`{}`) - request, _ := http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response := httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 400, response.Code, "OK response is expected") - - // test with invalid params - data = []byte(`{"bet_tag":"hi"}`) - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 400, response.Code, "OK response is expected") - - // test with valid params of invalid type - data = []byte(`{"pinned_tag":"hi", "auto_deploy: "true"}`) - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 400, response.Code, "OK response is expected") - - // test with invalid repoName - data = []byte(`{"pinned_tag":"v0.0.1"}`) - request, _ = http.NewRequest("POST", "/tags/nonexistent-repo", bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 400, response.Code, "OK response is expected") - - // test with invalid pinnedTag - data = []byte(`{"pinned_tag":"v0.5.0"}`) - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 400, response.Code, "OK response is expected") - - // test with valid pinnedTag - data = []byte(`{"pinned_tag":"v1.0.0"}`) - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 200, response.Code, "OK response is expected") - - // test with valid autoDeploy - data = []byte(`{"auto_deploy":true}`) - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 200, response.Code, "OK response is expected") - - // test with valid pinnedTag and pinnnedTag - data = []byte(`{"pinned_tag":"v1.0.0", "auto_deploy":true}`) - request, _ = http.NewRequest("POST", fmt.Sprintf("/tags/%s", te.TestRepoName), bytes.NewBuffer(data)) - response = httptest.NewRecorder() - router.ServeHTTP(response, request) - assert.Equal(t, 200, response.Code, "OK response is expected") -}