From 2e7daac7d02f152e3a47a9ded1857a852fdb6fc3 Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 18 Dec 2024 09:32:06 +0800 Subject: [PATCH 1/4] sync code component --- common/types/code.go | 45 +++++++++++++++++++++--------------------- common/types/repo.go | 20 +++++++++++++++++++ component/code.go | 3 +++ component/code_test.go | 13 ++++++------ 4 files changed, 53 insertions(+), 28 deletions(-) diff --git a/common/types/code.go b/common/types/code.go index bc1ff658..b90b8bdf 100644 --- a/common/types/code.go +++ b/common/types/code.go @@ -17,26 +17,27 @@ type UpdateCodeReq struct { } type Code struct { - ID int64 `json:"id"` - Name string `json:"name"` - Nickname string `json:"nickname"` - Description string `json:"description"` - Likes int64 `json:"likes"` - Downloads int64 `json:"downloads"` - Path string `json:"path"` - RepositoryID int64 `json:"repository_id"` - Repository Repository `json:"repository"` - Private bool `json:"private"` - User User `json:"user"` - Tags []RepoTag `json:"tags"` - DefaultBranch string `json:"default_branch"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - UserLikes bool `json:"user_likes"` - Source RepositorySource `json:"source"` - SyncStatus RepositorySyncStatus `json:"sync_status"` - License string `json:"license"` - CanWrite bool `json:"can_write"` - CanManage bool `json:"can_manage"` - Namespace *Namespace `json:"namespace"` + ID int64 `json:"id"` + Name string `json:"name"` + Nickname string `json:"nickname"` + Description string `json:"description"` + Likes int64 `json:"likes"` + Downloads int64 `json:"downloads"` + Path string `json:"path"` + RepositoryID int64 `json:"repository_id"` + Repository Repository `json:"repository"` + Private bool `json:"private"` + User User `json:"user"` + Tags []RepoTag `json:"tags"` + DefaultBranch string `json:"default_branch"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + UserLikes bool `json:"user_likes"` + Source RepositorySource `json:"source"` + SyncStatus RepositorySyncStatus `json:"sync_status"` + License string `json:"license"` + CanWrite bool `json:"can_write"` + CanManage bool `json:"can_manage"` + Namespace *Namespace `json:"namespace"` + SensitiveCheckStatus string `json:"sensitive_check_status"` } diff --git a/common/types/repo.go b/common/types/repo.go index 63c93013..ba387aac 100644 --- a/common/types/repo.go +++ b/common/types/repo.go @@ -9,6 +9,26 @@ type RepositorySource string type RepositorySyncStatus string type SensitiveCheckStatus int +// String returns a string representation of the sensitive check status. +// +// It returns one of "Fail", "Pending", "Pass", "Skip", "Exception", or "Unknown". +func (s SensitiveCheckStatus) String() string { + switch s { + case SensitiveCheckFail: + return "Fail" + case SensitiveCheckPending: + return "Pending" + case SensitiveCheckPass: + return "Pass" + case SensitiveCheckSkip: + return "Skip" + case SensitiveCheckException: + return "Exception" + default: + return "Unknown" + } +} + const ( ResTypeKey string = "hub-res-type" ResNameKey string = "hub-res-name" diff --git a/component/code.go b/component/code.go index 07f7a8b1..4db155e2 100644 --- a/component/code.go +++ b/component/code.go @@ -349,6 +349,9 @@ func (c *codeComponentImpl) Show(ctx context.Context, namespace, name, currentUs CanManage: permission.CanAdmin, Namespace: ns, } + if permission.CanAdmin { + resCode.SensitiveCheckStatus = code.Repository.SensitiveCheckStatus.String() + } return resCode, nil } diff --git a/component/code_test.go b/component/code_test.go index 4dff2019..56a9530e 100644 --- a/component/code_test.go +++ b/component/code_test.go @@ -166,12 +166,13 @@ func TestCodeComponent_Show(t *testing.T) { HTTPCloneURL: "/s/.git", SSHCloneURL: ":s/.git", }, - RepositoryID: 11, - Namespace: &types.Namespace{}, - Name: "name", - User: types.User{Username: "user"}, - CanManage: true, - UserLikes: true, + RepositoryID: 11, + Namespace: &types.Namespace{}, + Name: "name", + User: types.User{Username: "user"}, + CanManage: true, + UserLikes: true, + SensitiveCheckStatus: "Pending", }, data) } From 2d483be43d1b49a7e3ffeb3e77c999c5a37cc8e9 Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 18 Dec 2024 09:36:58 +0800 Subject: [PATCH 2/4] sync dataset component --- common/types/dataset.go | 49 +++++++++++++++++++------------------- component/dataset.go | 5 +++- component/git_http.go | 4 ++-- component/git_http_test.go | 2 +- component/repo.go | 4 ++-- component/repo_test.go | 4 ++-- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/common/types/dataset.go b/common/types/dataset.go index f81915f1..14462801 100644 --- a/common/types/dataset.go +++ b/common/types/dataset.go @@ -18,28 +18,29 @@ type UpdateDatasetReq struct { } type Dataset struct { - ID int64 `json:"id,omitempty"` - Name string `json:"name"` - Nickname string `json:"nickname"` - Description string `json:"description"` - Likes int64 `json:"likes"` - Downloads int64 `json:"downloads"` - Path string `json:"path"` - RepositoryID int64 `json:"repository_id"` - Repository Repository `json:"repository"` - Private bool `json:"private"` - User User `json:"user"` - Tags []RepoTag `json:"tags"` - Readme string `json:"readme"` - DefaultBranch string `json:"default_branch"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - UserLikes bool `json:"user_likes"` - Source RepositorySource `json:"source"` - SyncStatus RepositorySyncStatus `json:"sync_status"` - License string `json:"license"` - CanWrite bool `json:"can_write"` - CanManage bool `json:"can_manage"` - Namespace *Namespace `json:"namespace"` - MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` + ID int64 `json:"id,omitempty"` + Name string `json:"name"` + Nickname string `json:"nickname"` + Description string `json:"description"` + Likes int64 `json:"likes"` + Downloads int64 `json:"downloads"` + Path string `json:"path"` + RepositoryID int64 `json:"repository_id"` + Repository Repository `json:"repository"` + Private bool `json:"private"` + User User `json:"user"` + Tags []RepoTag `json:"tags"` + Readme string `json:"readme"` + DefaultBranch string `json:"default_branch"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + UserLikes bool `json:"user_likes"` + Source RepositorySource `json:"source"` + SyncStatus RepositorySyncStatus `json:"sync_status"` + License string `json:"license"` + CanWrite bool `json:"can_write"` + CanManage bool `json:"can_manage"` + Namespace *Namespace `json:"namespace"` + SensitiveCheckStatus string `json:"sensitive_check_status"` + MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` } diff --git a/component/dataset.go b/component/dataset.go index 302fbf68..875a2329 100644 --- a/component/dataset.go +++ b/component/dataset.go @@ -77,7 +77,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text const ( initCommitMessage = "initial commit" - ossFileExpireSeconds = 259200 * time.Second + ossFileExpire = 259200 * time.Second readmeFileName = "README.md" gitattributesFileName = ".gitattributes" ) @@ -473,6 +473,9 @@ func (c *datasetComponentImpl) Show(ctx context.Context, namespace, name, curren CanManage: permission.CanAdmin, Namespace: ns, } + if permission.CanAdmin { + resDataset.SensitiveCheckStatus = dataset.Repository.SensitiveCheckStatus.String() + } return resDataset, nil } diff --git a/component/git_http.go b/component/git_http.go index 4decc37a..d69ab5f4 100644 --- a/component/git_http.go +++ b/component/git_http.go @@ -281,7 +281,7 @@ func (c *gitHTTPComponentImpl) buildObjectResponse(ctx context.Context, req type var link *types.Link reqParams := make(url.Values) objectKey := path.Join("lfs", pointer.RelativePath()) - url, err := c.s3Client.PresignedGetObject(ctx, c.config.S3.Bucket, objectKey, ossFileExpireSeconds, reqParams) + url, err := c.s3Client.PresignedGetObject(ctx, c.config.S3.Bucket, objectKey, ossFileExpire, reqParams) if url != nil && err == nil { delete(header, "Authorization") link = &types.Link{Href: url.String(), Header: header} @@ -682,7 +682,7 @@ func (c *gitHTTPComponentImpl) LfsDownload(ctx context.Context, req types.Downlo // allow rename when download through content-disposition header reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", req.SaveAs)) } - signedUrl, err := c.s3Client.PresignedGetObject(ctx, c.config.S3.Bucket, objectKey, ossFileExpireSeconds, reqParams) + signedUrl, err := c.s3Client.PresignedGetObject(ctx, c.config.S3.Bucket, objectKey, ossFileExpire, reqParams) if err != nil { return nil, err } diff --git a/component/git_http_test.go b/component/git_http_test.go index 88d24547..25de03b4 100644 --- a/component/git_http_test.go +++ b/component/git_http_test.go @@ -476,7 +476,7 @@ func TestGitHTTPComponent_LfsDownload(t *testing.T) { reqParams := make(url.Values) reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", "sa")) url := &url.URL{Scheme: "http"} - gc.mocks.s3Client.EXPECT().PresignedGetObject(ctx, "", "lfs/oid", ossFileExpireSeconds, reqParams).Return(url, nil) + gc.mocks.s3Client.EXPECT().PresignedGetObject(ctx, "", "lfs/oid", ossFileExpire, reqParams).Return(url, nil) u, err := gc.LfsDownload(ctx, types.DownloadRequest{ Oid: "oid", diff --git a/component/repo.go b/component/repo.go index 95a73d01..9977780e 100644 --- a/component/repo.go +++ b/component/repo.go @@ -968,7 +968,7 @@ func (c *repoComponentImpl) DownloadFile(ctx context.Context, req *types.GetFile // allow rename when download through content-disposition header reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", req.SaveAs)) } - signedUrl, err := c.s3Client.PresignedGetObject(ctx, c.lfsBucket, objectKey, ossFileExpireSeconds, reqParams) + signedUrl, err := c.s3Client.PresignedGetObject(ctx, c.lfsBucket, objectKey, ossFileExpire, reqParams) if err != nil { return nil, 0, downloadUrl, err } @@ -1289,7 +1289,7 @@ func (c *repoComponentImpl) SDKDownloadFile(ctx context.Context, req *types.GetF // allow rename when download through content-disposition header reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", req.SaveAs)) } - signedUrl, err := c.s3Client.PresignedGetObject(ctx, c.lfsBucket, objectKey, ossFileExpireSeconds, reqParams) + signedUrl, err := c.s3Client.PresignedGetObject(ctx, c.lfsBucket, objectKey, ossFileExpire, reqParams) if err != nil { if err.Error() == ErrNotFoundMessage || err.Error() == ErrGetContentsOrList { return nil, 0, downloadUrl, ErrNotFound diff --git a/component/repo_test.go b/component/repo_test.go index 3959b7c3..c19df4d2 100644 --- a/component/repo_test.go +++ b/component/repo_test.go @@ -652,7 +652,7 @@ func TestRepoComponent_DownloadFile(t *testing.T) { reqParams := make(url.Values) reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", "zzz")) repo.mocks.s3Client.EXPECT().PresignedGetObject( - ctx, repo.lfsBucket, "lfs/path", ossFileExpireSeconds, reqParams, + ctx, repo.lfsBucket, "lfs/path", ossFileExpire, reqParams, ).Return(&url.URL{Path: "foobar"}, nil) } else { repo.mocks.gitServer.EXPECT().GetRepoFileReader(ctx, gitserver.GetRepoInfoByPathReq{ @@ -765,7 +765,7 @@ func TestRepoComponent_SDKDownloadFile(t *testing.T) { reqParams := make(url.Values) reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", "zzz")) repo.mocks.s3Client.EXPECT().PresignedGetObject( - ctx, repo.lfsBucket, "lfs/qqq", ossFileExpireSeconds, reqParams, + ctx, repo.lfsBucket, "lfs/qqq", ossFileExpire, reqParams, ).Return(&url.URL{Path: "foobar"}, nil) } else { repo.mocks.gitServer.EXPECT().GetRepoFileReader(ctx, gitserver.GetRepoInfoByPathReq{ From c3c320b4613b708f3833376b0de5bb765f157c95 Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 18 Dec 2024 09:41:35 +0800 Subject: [PATCH 3/4] sync list component --- component/list.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/component/list.go b/component/list.go index 2fcccdde..a906522b 100644 --- a/component/list.go +++ b/component/list.go @@ -33,7 +33,7 @@ func (c *listComponentImpl) ListModelsByPath(ctx context.Context, req *types.Lis models, err := c.modelStore.ListByPath(ctx, req.Paths) if err != nil { - slog.Error("error listing models by path", "error", err, slog.Any("paths", req.Paths)) + slog.Error("error listing models by path: %v", slog.Any("error", err), slog.Any("paths", req.Paths)) return nil, err } for _, model := range models { @@ -69,7 +69,7 @@ func (c *listComponentImpl) ListDatasetsByPath(ctx context.Context, req *types.L datasets, err := c.datasetStore.ListByPath(ctx, req.Paths) if err != nil { - slog.Error("error listing datasets by path", "error", err, slog.Any("paths", req.Paths)) + slog.Error("error listing datasets by path: %v", slog.Any("error", err), slog.Any("paths", req.Paths)) return nil, err } for _, dataset := range datasets { From 3445076cecf9c626a29be4a2bc2c3928e4b020bb Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 18 Dec 2024 09:48:06 +0800 Subject: [PATCH 4/4] sync mirror component --- .../store/database/mock_MirrorStore.go | 91 ++++- api/handler/mirror.go | 3 +- builder/store/database/mirror.go | 39 +- builder/store/database/mirror_test.go | 13 +- common/types/mirror.go | 5 + component/mirror.go | 74 ++-- component/mirror_test.go | 355 ++++++++++++++++++ 7 files changed, 534 insertions(+), 46 deletions(-) create mode 100644 component/mirror_test.go diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go index 4f47d431..48656ec1 100644 --- a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go @@ -429,9 +429,9 @@ func (_c *MockMirrorStore_Finished_Call) RunAndReturn(run func(context.Context) return _c } -// IndexWithPagination provides a mock function with given fields: ctx, per, page -func (_m *MockMirrorStore) IndexWithPagination(ctx context.Context, per int, page int) ([]database.Mirror, int, error) { - ret := _m.Called(ctx, per, page) +// IndexWithPagination provides a mock function with given fields: ctx, per, page, search +func (_m *MockMirrorStore) IndexWithPagination(ctx context.Context, per int, page int, search string) ([]database.Mirror, int, error) { + ret := _m.Called(ctx, per, page, search) if len(ret) == 0 { panic("no return value specified for IndexWithPagination") @@ -440,25 +440,25 @@ func (_m *MockMirrorStore) IndexWithPagination(ctx context.Context, per int, pag var r0 []database.Mirror var r1 int var r2 error - if rf, ok := ret.Get(0).(func(context.Context, int, int) ([]database.Mirror, int, error)); ok { - return rf(ctx, per, page) + if rf, ok := ret.Get(0).(func(context.Context, int, int, string) ([]database.Mirror, int, error)); ok { + return rf(ctx, per, page, search) } - if rf, ok := ret.Get(0).(func(context.Context, int, int) []database.Mirror); ok { - r0 = rf(ctx, per, page) + if rf, ok := ret.Get(0).(func(context.Context, int, int, string) []database.Mirror); ok { + r0 = rf(ctx, per, page, search) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]database.Mirror) } } - if rf, ok := ret.Get(1).(func(context.Context, int, int) int); ok { - r1 = rf(ctx, per, page) + if rf, ok := ret.Get(1).(func(context.Context, int, int, string) int); ok { + r1 = rf(ctx, per, page, search) } else { r1 = ret.Get(1).(int) } - if rf, ok := ret.Get(2).(func(context.Context, int, int) error); ok { - r2 = rf(ctx, per, page) + if rf, ok := ret.Get(2).(func(context.Context, int, int, string) error); ok { + r2 = rf(ctx, per, page, search) } else { r2 = ret.Error(2) } @@ -475,13 +475,14 @@ type MockMirrorStore_IndexWithPagination_Call struct { // - ctx context.Context // - per int // - page int -func (_e *MockMirrorStore_Expecter) IndexWithPagination(ctx interface{}, per interface{}, page interface{}) *MockMirrorStore_IndexWithPagination_Call { - return &MockMirrorStore_IndexWithPagination_Call{Call: _e.mock.On("IndexWithPagination", ctx, per, page)} +// - search string +func (_e *MockMirrorStore_Expecter) IndexWithPagination(ctx interface{}, per interface{}, page interface{}, search interface{}) *MockMirrorStore_IndexWithPagination_Call { + return &MockMirrorStore_IndexWithPagination_Call{Call: _e.mock.On("IndexWithPagination", ctx, per, page, search)} } -func (_c *MockMirrorStore_IndexWithPagination_Call) Run(run func(ctx context.Context, per int, page int)) *MockMirrorStore_IndexWithPagination_Call { +func (_c *MockMirrorStore_IndexWithPagination_Call) Run(run func(ctx context.Context, per int, page int, search string)) *MockMirrorStore_IndexWithPagination_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int), args[2].(int)) + run(args[0].(context.Context), args[1].(int), args[2].(int), args[3].(string)) }) return _c } @@ -491,7 +492,7 @@ func (_c *MockMirrorStore_IndexWithPagination_Call) Return(mirrors []database.Mi return _c } -func (_c *MockMirrorStore_IndexWithPagination_Call) RunAndReturn(run func(context.Context, int, int) ([]database.Mirror, int, error)) *MockMirrorStore_IndexWithPagination_Call { +func (_c *MockMirrorStore_IndexWithPagination_Call) RunAndReturn(run func(context.Context, int, int, string) ([]database.Mirror, int, error)) *MockMirrorStore_IndexWithPagination_Call { _c.Call.Return(run) return _c } @@ -728,6 +729,64 @@ func (_c *MockMirrorStore_PushedMirror_Call) RunAndReturn(run func(context.Conte return _c } +// StatusCount provides a mock function with given fields: ctx +func (_m *MockMirrorStore) StatusCount(ctx context.Context) ([]database.MirrorStatusCount, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for StatusCount") + } + + var r0 []database.MirrorStatusCount + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]database.MirrorStatusCount, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []database.MirrorStatusCount); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]database.MirrorStatusCount) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMirrorStore_StatusCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StatusCount' +type MockMirrorStore_StatusCount_Call struct { + *mock.Call +} + +// StatusCount is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockMirrorStore_Expecter) StatusCount(ctx interface{}) *MockMirrorStore_StatusCount_Call { + return &MockMirrorStore_StatusCount_Call{Call: _e.mock.On("StatusCount", ctx)} +} + +func (_c *MockMirrorStore_StatusCount_Call) Run(run func(ctx context.Context)) *MockMirrorStore_StatusCount_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockMirrorStore_StatusCount_Call) Return(_a0 []database.MirrorStatusCount, _a1 error) *MockMirrorStore_StatusCount_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMirrorStore_StatusCount_Call) RunAndReturn(run func(context.Context) ([]database.MirrorStatusCount, error)) *MockMirrorStore_StatusCount_Call { + _c.Call.Return(run) + return _c +} + // ToSyncLfs provides a mock function with given fields: ctx func (_m *MockMirrorStore) ToSyncLfs(ctx context.Context) ([]database.Mirror, error) { ret := _m.Called(ctx) diff --git a/api/handler/mirror.go b/api/handler/mirror.go index 36d35d46..bc74824b 100644 --- a/api/handler/mirror.go +++ b/api/handler/mirror.go @@ -123,7 +123,8 @@ func (h *MirrorHandler) Index(ctx *gin.Context) { httpbase.UnauthorizedError(ctx, component.ErrUserNotFound) return } - repos, total, err := h.mc.Index(ctx, currentUser, per, page) + search := ctx.Query("search") + repos, total, err := h.mc.Index(ctx, currentUser, per, page, search) if err != nil { slog.Error("failed to get mirror repos", slog.Any("error", err)) httpbase.ServerError(ctx, err) diff --git a/builder/store/database/mirror.go b/builder/store/database/mirror.go index d9ccd448..2d266d02 100644 --- a/builder/store/database/mirror.go +++ b/builder/store/database/mirror.go @@ -3,6 +3,7 @@ package database import ( "context" "fmt" + "strings" "time" "github.com/uptrace/bun" @@ -31,7 +32,8 @@ type MirrorStore interface { Finished(ctx context.Context) ([]Mirror, error) ToSyncRepo(ctx context.Context) ([]Mirror, error) ToSyncLfs(ctx context.Context) ([]Mirror, error) - IndexWithPagination(ctx context.Context, per, page int) (mirrors []Mirror, count int, err error) + IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error) + StatusCount(ctx context.Context) ([]MirrorStatusCount, error) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error } @@ -76,6 +78,11 @@ type Mirror struct { times } +type MirrorStatusCount struct { + Status types.MirrorTaskStatus `bun:"status"` + Count int `bun:"count"` +} + func (s *mirrorStoreImpl) IsExist(ctx context.Context, repoID int64) (exists bool, err error) { var mirror Mirror exists, err = s.db.Operator.Core. @@ -90,7 +97,8 @@ func (s *mirrorStoreImpl) IsRepoExist(ctx context.Context, repoType types.Reposi exists, err = s.db.Operator.Core. NewSelect(). Model(&repo). - Where("git_path=?", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)). + Where("path=?", fmt.Sprintf("%s/%s", namespace, name)). + Where("repository_type=?", repoType). Exists(ctx) return } @@ -273,7 +281,13 @@ func (s *mirrorStoreImpl) ToSyncRepo(ctx context.Context) ([]Mirror, error) { var mirrors []Mirror err := s.db.Operator.Core.NewSelect(). Model(&mirrors). - Where("next_execution_timestamp < ? or status in (?,?,?)", time.Now(), types.MirrorIncomplete, types.MirrorFailed, types.MirrorWaiting). + Where( + "next_execution_timestamp < ? or status in (?,?,?,?)", + time.Now(), + types.MirrorIncomplete, + types.MirrorFailed, + types.MirrorWaiting, + types.MirrorRunning). Scan(ctx) if err != nil { return nil, err @@ -293,11 +307,17 @@ func (s *mirrorStoreImpl) ToSyncLfs(ctx context.Context) ([]Mirror, error) { return mirrors, nil } -func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int) (mirrors []Mirror, count int, err error) { +func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error) { q := s.db.Operator.Core.NewSelect(). Model(&mirrors). Relation("Repository"). Relation("MirrorSource") + if search != "" { + q = q.Where("LOWER(mirror.source_url) like ? or LOWER(mirror.local_repo_path) like ?", + fmt.Sprintf("%%%s%%", strings.ToLower(search)), + fmt.Sprintf("%%%s%%", strings.ToLower(search)), + ) + } count, err = q.Count(ctx) if err != nil { return @@ -313,6 +333,17 @@ func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int return } +func (s *mirrorStoreImpl) StatusCount(ctx context.Context) ([]MirrorStatusCount, error) { + var statusCounts []MirrorStatusCount + err := s.db.Operator.Core.NewSelect(). + Model((*Mirror)(nil)). + Column("status"). + ColumnExpr("COUNT(*) AS count"). + Group("status"). + Scan(ctx, &statusCounts) + return statusCounts, err +} + func (s *mirrorStoreImpl) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error { err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { _, err := tx.NewUpdate().Model(mirror).WherePK().Exec(ctx) diff --git a/builder/store/database/mirror_test.go b/builder/store/database/mirror_test.go index ab056121..6175ebc3 100644 --- a/builder/store/database/mirror_test.go +++ b/builder/store/database/mirror_test.go @@ -51,6 +51,7 @@ func TestMirrorStore_CRUD(t *testing.T) { RepositoryType: types.ModelRepo, GitPath: "models_ns/n", Name: "repo", + Path: "ns/n", } err = db.Core.NewInsert().Model(repo).Scan(ctx, repo) require.Nil(t, err) @@ -194,7 +195,7 @@ func TestMirrorStore_ToSync(t *testing.T) { for _, m := range ms { names = append(names, m.Interval) } - require.ElementsMatch(t, []string{"m1", "m3", "m6", "m7"}, names) + require.ElementsMatch(t, []string{"m1", "m3", "m5", "m6", "m7"}, names) ms, err = store.ToSyncLfs(ctx) require.Nil(t, err) @@ -222,7 +223,7 @@ func TestMirrorStore_IndexWithPagination(t *testing.T) { require.Nil(t, err) } - ms, count, err := store.IndexWithPagination(ctx, 10, 1) + ms, count, err := store.IndexWithPagination(ctx, 10, 1, "foo") require.Nil(t, err) names := []string{} for _, m := range ms { @@ -250,4 +251,12 @@ func TestMirrorStore_StatusCount(t *testing.T) { require.Nil(t, err) } + cs, err := store.StatusCount(ctx) + require.Nil(t, err) + require.Equal(t, 2, len(cs)) + require.ElementsMatch(t, []database.MirrorStatusCount{ + {types.MirrorFailed, 2}, + {types.MirrorFinished, 1}, + }, cs) + } diff --git a/common/types/mirror.go b/common/types/mirror.go index 294bed0c..d079597c 100644 --- a/common/types/mirror.go +++ b/common/types/mirror.go @@ -135,3 +135,8 @@ type Mirror struct { type MirrorSource struct { SourceName string `json:"source_name"` } + +type MirrorStatusCount struct { + Status MirrorTaskStatus + Count int +} diff --git a/component/mirror.go b/component/mirror.go index 050ae92e..67aa642b 100644 --- a/component/mirror.go +++ b/component/mirror.go @@ -48,7 +48,8 @@ type MirrorComponent interface { CreateMirrorRepo(ctx context.Context, req types.CreateMirrorRepoReq) (*database.Mirror, error) CheckMirrorProgress(ctx context.Context) error Repos(ctx context.Context, currentUser string, per, page int) ([]types.MirrorRepo, int, error) - Index(ctx context.Context, currentUser string, per, page int) ([]types.Mirror, int, error) + Index(ctx context.Context, currentUser string, per, page int, search string) ([]types.Mirror, int, error) + Statistics(ctx context.Context, currentUser string) ([]types.MirrorStatusCount, error) } func NewMirrorComponent(config *config.Config) (MirrorComponent, error) { @@ -66,7 +67,8 @@ func NewMirrorComponent(config *config.Config) (MirrorComponent, error) { if err != nil { return nil, fmt.Errorf("failed to get priority queue: %v", err) } - c.repoComp, err = NewRepoComponent(config) + + c.repoComp, err = NewRepoComponentImpl(config) if err != nil { return nil, fmt.Errorf("fail to create repo component,error:%w", err) } @@ -246,6 +248,7 @@ func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.Cr mirror.LocalRepoPath = fmt.Sprintf("%s_%s_%s_%s", mirrorSource.SourceName, req.RepoType, req.SourceNamespace, req.SourceName) mirror.SourceRepoPath = fmt.Sprintf("%s/%s", req.SourceNamespace, req.SourceName) mirror.Priority = types.HighMirrorPriority + var taskId int64 if c.config.GitServer.Type == types.GitServerTypeGitea { taskId, err = c.mirrorServer.CreateMirrorRepo(ctx, mirrorserver.CreateMirrorRepoReq{ @@ -262,14 +265,12 @@ func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.Cr return nil, fmt.Errorf("failed to create push mirror in mirror server: %v", err) } } - mirror.MirrorTaskID = taskId reqMirror, err := c.mirrorStore.Create(ctx, &mirror) if err != nil { return nil, fmt.Errorf("failed to create mirror") } - if c.config.GitServer.Type == types.GitServerTypeGitaly { c.mq.PushRepoMirror(&queue.MirrorTask{ MirrorID: reqMirror.ID, @@ -283,6 +284,7 @@ func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.Cr } return reqMirror, nil + } func (m *mirrorComponentImpl) mapNamespaceAndName(sourceNamespace string) string { @@ -620,7 +622,7 @@ func (c *mirrorComponentImpl) Repos(ctx context.Context, currentUser string, per return mirrorRepos, total, nil } -func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per, page int) ([]types.Mirror, int, error) { +func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per, page int, search string) ([]types.Mirror, int, error) { var mirrorsResp []types.Mirror user, err := c.userStore.FindByUsername(ctx, currentUser) if err != nil { @@ -629,28 +631,54 @@ func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per if !user.CanAdmin() { return nil, 0, errors.New("user does not have admin permission") } - mirrors, total, err := c.mirrorStore.IndexWithPagination(ctx, per, page) + mirrors, total, err := c.mirrorStore.IndexWithPagination(ctx, per, page, search) if err != nil { return nil, 0, fmt.Errorf("failed to get mirror mirrors: %v", err) } for _, mirror := range mirrors { - mirrorsResp = append(mirrorsResp, types.Mirror{ - SourceUrl: mirror.SourceUrl, - MirrorSource: types.MirrorSource{ - SourceName: mirror.MirrorSource.SourceName, - }, - Username: mirror.Username, - AccessToken: mirror.AccessToken, - PushUrl: mirror.PushUrl, - PushUsername: mirror.PushUsername, - PushAccessToken: mirror.PushAccessToken, - LastUpdatedAt: mirror.LastUpdatedAt, - SourceRepoPath: mirror.SourceRepoPath, - LocalRepoPath: fmt.Sprintf("%ss/%s", mirror.Repository.RepositoryType, mirror.Repository.Path), - LastMessage: mirror.LastMessage, - Status: mirror.Status, - Progress: mirror.Progress, - }) + if mirror.Repository != nil { + mirrorsResp = append(mirrorsResp, types.Mirror{ + SourceUrl: mirror.SourceUrl, + MirrorSource: types.MirrorSource{ + SourceName: mirror.MirrorSource.SourceName, + }, + Username: mirror.Username, + AccessToken: mirror.AccessToken, + PushUrl: mirror.PushUrl, + PushUsername: mirror.PushUsername, + PushAccessToken: mirror.PushAccessToken, + LastUpdatedAt: mirror.LastUpdatedAt, + SourceRepoPath: mirror.SourceRepoPath, + LocalRepoPath: fmt.Sprintf("%ss/%s", mirror.Repository.RepositoryType, mirror.Repository.Path), + LastMessage: mirror.LastMessage, + Status: mirror.Status, + Progress: mirror.Progress, + }) + } } return mirrorsResp, total, nil } + +func (c *mirrorComponentImpl) Statistics(ctx context.Context, currentUser string) ([]types.MirrorStatusCount, error) { + var scs []types.MirrorStatusCount + user, err := c.userStore.FindByUsername(ctx, currentUser) + if err != nil { + return nil, errors.New("user does not exist") + } + if !user.CanAdmin() { + return nil, errors.New("user does not have admin permission") + } + statusCounts, err := c.mirrorStore.StatusCount(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get mirror statistics: %v", err) + } + + for _, statusCount := range statusCounts { + scs = append(scs, types.MirrorStatusCount{ + Status: statusCount.Status, + Count: statusCount.Count, + }) + } + + return scs, nil +} diff --git a/component/mirror_test.go b/component/mirror_test.go new file mode 100644 index 00000000..375002c3 --- /dev/null +++ b/component/mirror_test.go @@ -0,0 +1,355 @@ +package component + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "opencsg.com/csghub-server/builder/git/gitserver" + "opencsg.com/csghub-server/builder/git/mirrorserver" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/mirror/queue" +) + +func TestMirrorComponent_CreatePushMirrorForFinishedMirrorTask(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.MirrorMock().EXPECT().NoPushMirror(ctx).Return([]database.Mirror{ + {MirrorTaskID: 1}, + {MirrorTaskID: 2, LocalRepoPath: "foo"}, + }, nil) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(1)).Return( + &mirrorserver.MirrorTaskInfo{}, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(2)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusFinished, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().CreatePushMirror(ctx, mirrorserver.CreatePushMirrorReq{ + Name: "foo", + Interval: "8h", + }).Return(nil) + mc.mocks.stores.MirrorMock().EXPECT().Update(ctx, &database.Mirror{ + MirrorTaskID: 2, LocalRepoPath: "foo", PushMirrorCreated: true, + }).Return(nil) + + err := mc.CreatePushMirrorForFinishedMirrorTask(ctx) + require.Nil(t, err) +} + +func TestMirrorComponent_CreateMirrorRepo(t *testing.T) { + + cases := []struct { + repoType types.RepositoryType + gitea bool + }{ + {types.ModelRepo, false}, + {types.DatasetRepo, false}, + {types.CodeRepo, false}, + {types.CodeRepo, true}, + } + + for _, c := range cases { + t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) { + + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + req := types.CreateMirrorRepoReq{ + SourceNamespace: "sns", + SourceName: "sn", + RepoType: c.repoType, + CurrentUser: "user", + } + + if c.gitea { + mc.config.GitServer.Type = types.GitServerTypeGitea + } else { + mc.config.GitServer.Type = types.GitServerTypeGitaly + } + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + repo := &database.Repository{} + mc.mocks.stores.RepoMock().EXPECT().FindByPath( + ctx, req.RepoType, "AIWizards", "sn", + ).Return( + repo, nil, + ) + mc.mocks.stores.RepoMock().EXPECT().FindByPath( + ctx, req.RepoType, "AIWizards", "sns_sn", + ).Return( + nil, sql.ErrNoRows, + ) + mc.mocks.stores.NamespaceMock().EXPECT().FindByPath( + ctx, "AIWizards", + ).Return(database.Namespace{ + User: database.User{Username: "user"}, + }, nil) + mc.mocks.components.repo.EXPECT().CreateRepo(ctx, types.CreateRepoReq{ + Username: "user", + Namespace: "AIWizards", + Name: "sns_sn", + Nickname: "sns_sn", + Description: req.Description, + Private: true, + License: req.License, + DefaultBranch: req.DefaultBranch, + RepoType: req.RepoType, + }).Return(&gitserver.CreateRepoResp{}, &database.Repository{}, nil) + switch req.RepoType { + case types.ModelRepo: + mc.mocks.stores.ModelMock().EXPECT().Create(ctx, database.Model{ + Repository: repo, + RepositoryID: repo.ID, + }).Return(nil, nil) + case types.DatasetRepo: + mc.mocks.stores.DatasetMock().EXPECT().Create(ctx, database.Dataset{ + Repository: repo, + RepositoryID: repo.ID, + }).Return(nil, nil) + case types.CodeRepo: + mc.mocks.stores.CodeMock().EXPECT().Create(ctx, database.Code{ + Repository: repo, + RepositoryID: repo.ID, + }).Return(nil, nil) + } + mc.mocks.stores.GitServerAccessTokenMock().EXPECT().FindByType(ctx, "git").Return( + []database.GitServerAccessToken{ + {}, + }, nil, + ) + mc.mocks.stores.MirrorSourceMock().EXPECT().Get(ctx, int64(0)).Return( + &database.MirrorSource{}, nil, + ) + if c.gitea { + mc.mocks.mirrorServer.EXPECT().CreateMirrorRepo(ctx, mirrorserver.CreateMirrorRepoReq{ + Name: "_code_sns_sn", + Namespace: "root", + Private: false, + SyncLfs: req.SyncLfs, + }).Return(123, nil) + } + reqMirror := &database.Mirror{ + ID: 1, + Priority: types.HighMirrorPriority, + } + localRepoPath := "" + switch req.RepoType { + case types.ModelRepo: + localRepoPath = "_model_sns_sn" + case types.DatasetRepo: + localRepoPath = "_dataset_sns_sn" + case types.CodeRepo: + localRepoPath = "_code_sns_sn" + } + + cm := &database.Mirror{ + Username: "sns", + PushUsername: "root", + SourceRepoPath: "sns/sn", + LocalRepoPath: localRepoPath, + Priority: types.HighMirrorPriority, + Repository: &database.Repository{}, + } + if c.gitea { + cm.MirrorTaskID = 123 + } + mc.mocks.stores.MirrorMock().EXPECT().Create(ctx, cm).Return( + reqMirror, nil, + ) + if !c.gitea { + mc.mocks.mirrorQueue.EXPECT().PushRepoMirror(&queue.MirrorTask{ + MirrorID: reqMirror.ID, + Priority: queue.PriorityMap[reqMirror.Priority], + }) + mc.mocks.stores.MirrorMock().EXPECT().Update(ctx, reqMirror).Return(nil) + } + + m, err := mc.CreateMirrorRepo(ctx, req) + require.Nil(t, err) + require.Equal(t, reqMirror, m) + + }) + } + +} + +func TestMirrorComponent_CheckMirrorProgress(t *testing.T) { + + for _, saas := range []bool{false, true} { + t.Run(fmt.Sprintf("saas %v", saas), func(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + mc.saas = saas + + mirrors := []database.Mirror{ + { + ID: 1, MirrorTaskID: 11, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + { + ID: 2, MirrorTaskID: 12, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + { + ID: 3, MirrorTaskID: 13, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + { + ID: 4, MirrorTaskID: 14, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + } + mc.mocks.stores.MirrorMock().EXPECT().Unfinished(ctx).Return(mirrors, nil) + + if saas { + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(11)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusQueued, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(12)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusRunning, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(13)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusFailed, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(14)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusFinished, + }, nil, + ) + } else { + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(11)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusQueued, + }, nil, + ) + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(12)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusRunning, + }, nil, + ) + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(13)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusFailed, + }, nil, + ) + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(14)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusFinished, + }, nil, + ) + } + mirrors[0].Status = types.MirrorWaiting + mirrors[1].Status = types.MirrorRunning + mirrors[1].Progress = 100 + mirrors[2].Status = types.MirrorFailed + mirrors[3].Status = types.MirrorFinished + mirrors[3].Progress = 100 + mc.mocks.gitServer.EXPECT().GetRepo(ctx, gitserver.GetRepoReq{ + Namespace: "foo", + Name: "bar", + RepoType: types.ModelRepo, + }).Return(&gitserver.CreateRepoResp{}, nil) + for _, m := range mirrors { + m.Repository.SyncStatus = mirrorStatusAndRepoSyncStatusMapping[m.Status] + mv := m + mc.mocks.stores.MirrorMock().EXPECT().Update(ctx, &mv).Return(nil).Once() + mc.mocks.stores.RepoMock().EXPECT().UpdateRepo( + ctx, database.Repository{ + ID: 111, + Path: "foo/bar", + RepositoryType: types.ModelRepo, + SyncStatus: mirrorStatusAndRepoSyncStatusMapping[mv.Status], + }, + ).Return(nil, nil).Once() + } + mc.mocks.gitServer.EXPECT().GetRepoFileTree( + mock.Anything, gitserver.GetRepoInfoByPathReq{ + Namespace: "foo", Name: "bar", RepoType: "model"}, + ).Return([]*types.File{{Name: "foo.go"}}, nil) + + err := mc.CheckMirrorProgress(ctx) + require.Nil(t, err) + }) + } + +} + +func TestMirrorComponent_Repos(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + mc.mocks.stores.RepoMock().EXPECT().WithMirror(ctx, 10, 1).Return([]database.Repository{ + {Path: "foo", SyncStatus: types.SyncStatusCompleted, RepositoryType: types.ModelRepo}, + }, 100, nil) + + data, total, err := mc.Repos(ctx, "user", 10, 1) + require.Nil(t, err) + require.Equal(t, 100, total) + require.Equal(t, []types.MirrorRepo{ + {Path: "foo", SyncStatus: types.SyncStatusCompleted, RepoType: types.ModelRepo}, + }, data) +} + +func TestMirrorComponent_Index(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + mc.mocks.stores.MirrorMock().EXPECT().IndexWithPagination(ctx, 10, 1, "foo").Return( + []database.Mirror{{Username: "user", LastMessage: "msg", Repository: &database.Repository{}}}, 100, nil, + ) + + data, total, err := mc.Index(ctx, "user", 10, 1, "foo") + require.Nil(t, err) + require.Equal(t, 100, total) + require.Equal(t, []types.Mirror{ + {Username: "user", LastMessage: "msg", LocalRepoPath: "s/"}, + }, data) +} + +func TestMirrorComponent_Statistic(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + mc.mocks.stores.MirrorMock().EXPECT().StatusCount(ctx).Return([]database.MirrorStatusCount{ + {Status: types.MirrorFinished, Count: 100}, + }, nil) + + s, err := mc.Statistics(ctx, "user") + require.Nil(t, err) + require.Equal(t, []types.MirrorStatusCount{ + {Status: types.MirrorFinished, Count: 100}, + }, s) + +}