From 67007de107dbc58e48cf81eb4dad769465ed273a Mon Sep 17 00:00:00 2001 From: yiling Date: Wed, 18 Dec 2024 09:48:06 +0800 Subject: [PATCH] sync mirror component --- .../store/database/mock_MirrorStore.go | 91 ++++- api/handler/mirror.go | 3 +- builder/store/database/mirror.go | 39 +- builder/store/database/mirror_test.go | 13 +- common/types/mirror.go | 5 + component/mirror.go | 74 ++-- component/mirror_test.go | 355 ++++++++++++++++++ 7 files changed, 534 insertions(+), 46 deletions(-) create mode 100644 component/mirror_test.go diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go index 4f47d431..48656ec1 100644 --- a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go @@ -429,9 +429,9 @@ func (_c *MockMirrorStore_Finished_Call) RunAndReturn(run func(context.Context) return _c } -// IndexWithPagination provides a mock function with given fields: ctx, per, page -func (_m *MockMirrorStore) IndexWithPagination(ctx context.Context, per int, page int) ([]database.Mirror, int, error) { - ret := _m.Called(ctx, per, page) +// IndexWithPagination provides a mock function with given fields: ctx, per, page, search +func (_m *MockMirrorStore) IndexWithPagination(ctx context.Context, per int, page int, search string) ([]database.Mirror, int, error) { + ret := _m.Called(ctx, per, page, search) if len(ret) == 0 { panic("no return value specified for IndexWithPagination") @@ -440,25 +440,25 @@ func (_m *MockMirrorStore) IndexWithPagination(ctx context.Context, per int, pag var r0 []database.Mirror var r1 int var r2 error - if rf, ok := ret.Get(0).(func(context.Context, int, int) ([]database.Mirror, int, error)); ok { - return rf(ctx, per, page) + if rf, ok := ret.Get(0).(func(context.Context, int, int, string) ([]database.Mirror, int, error)); ok { + return rf(ctx, per, page, search) } - if rf, ok := ret.Get(0).(func(context.Context, int, int) []database.Mirror); ok { - r0 = rf(ctx, per, page) + if rf, ok := ret.Get(0).(func(context.Context, int, int, string) []database.Mirror); ok { + r0 = rf(ctx, per, page, search) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]database.Mirror) } } - if rf, ok := ret.Get(1).(func(context.Context, int, int) int); ok { - r1 = rf(ctx, per, page) + if rf, ok := ret.Get(1).(func(context.Context, int, int, string) int); ok { + r1 = rf(ctx, per, page, search) } else { r1 = ret.Get(1).(int) } - if rf, ok := ret.Get(2).(func(context.Context, int, int) error); ok { - r2 = rf(ctx, per, page) + if rf, ok := ret.Get(2).(func(context.Context, int, int, string) error); ok { + r2 = rf(ctx, per, page, search) } else { r2 = ret.Error(2) } @@ -475,13 +475,14 @@ type MockMirrorStore_IndexWithPagination_Call struct { // - ctx context.Context // - per int // - page int -func (_e *MockMirrorStore_Expecter) IndexWithPagination(ctx interface{}, per interface{}, page interface{}) *MockMirrorStore_IndexWithPagination_Call { - return &MockMirrorStore_IndexWithPagination_Call{Call: _e.mock.On("IndexWithPagination", ctx, per, page)} +// - search string +func (_e *MockMirrorStore_Expecter) IndexWithPagination(ctx interface{}, per interface{}, page interface{}, search interface{}) *MockMirrorStore_IndexWithPagination_Call { + return &MockMirrorStore_IndexWithPagination_Call{Call: _e.mock.On("IndexWithPagination", ctx, per, page, search)} } -func (_c *MockMirrorStore_IndexWithPagination_Call) Run(run func(ctx context.Context, per int, page int)) *MockMirrorStore_IndexWithPagination_Call { +func (_c *MockMirrorStore_IndexWithPagination_Call) Run(run func(ctx context.Context, per int, page int, search string)) *MockMirrorStore_IndexWithPagination_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int), args[2].(int)) + run(args[0].(context.Context), args[1].(int), args[2].(int), args[3].(string)) }) return _c } @@ -491,7 +492,7 @@ func (_c *MockMirrorStore_IndexWithPagination_Call) Return(mirrors []database.Mi return _c } -func (_c *MockMirrorStore_IndexWithPagination_Call) RunAndReturn(run func(context.Context, int, int) ([]database.Mirror, int, error)) *MockMirrorStore_IndexWithPagination_Call { +func (_c *MockMirrorStore_IndexWithPagination_Call) RunAndReturn(run func(context.Context, int, int, string) ([]database.Mirror, int, error)) *MockMirrorStore_IndexWithPagination_Call { _c.Call.Return(run) return _c } @@ -728,6 +729,64 @@ func (_c *MockMirrorStore_PushedMirror_Call) RunAndReturn(run func(context.Conte return _c } +// StatusCount provides a mock function with given fields: ctx +func (_m *MockMirrorStore) StatusCount(ctx context.Context) ([]database.MirrorStatusCount, error) { + ret := _m.Called(ctx) + + if len(ret) == 0 { + panic("no return value specified for StatusCount") + } + + var r0 []database.MirrorStatusCount + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]database.MirrorStatusCount, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []database.MirrorStatusCount); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]database.MirrorStatusCount) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMirrorStore_StatusCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StatusCount' +type MockMirrorStore_StatusCount_Call struct { + *mock.Call +} + +// StatusCount is a helper method to define mock.On call +// - ctx context.Context +func (_e *MockMirrorStore_Expecter) StatusCount(ctx interface{}) *MockMirrorStore_StatusCount_Call { + return &MockMirrorStore_StatusCount_Call{Call: _e.mock.On("StatusCount", ctx)} +} + +func (_c *MockMirrorStore_StatusCount_Call) Run(run func(ctx context.Context)) *MockMirrorStore_StatusCount_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *MockMirrorStore_StatusCount_Call) Return(_a0 []database.MirrorStatusCount, _a1 error) *MockMirrorStore_StatusCount_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMirrorStore_StatusCount_Call) RunAndReturn(run func(context.Context) ([]database.MirrorStatusCount, error)) *MockMirrorStore_StatusCount_Call { + _c.Call.Return(run) + return _c +} + // ToSyncLfs provides a mock function with given fields: ctx func (_m *MockMirrorStore) ToSyncLfs(ctx context.Context) ([]database.Mirror, error) { ret := _m.Called(ctx) diff --git a/api/handler/mirror.go b/api/handler/mirror.go index 36d35d46..bc74824b 100644 --- a/api/handler/mirror.go +++ b/api/handler/mirror.go @@ -123,7 +123,8 @@ func (h *MirrorHandler) Index(ctx *gin.Context) { httpbase.UnauthorizedError(ctx, component.ErrUserNotFound) return } - repos, total, err := h.mc.Index(ctx, currentUser, per, page) + search := ctx.Query("search") + repos, total, err := h.mc.Index(ctx, currentUser, per, page, search) if err != nil { slog.Error("failed to get mirror repos", slog.Any("error", err)) httpbase.ServerError(ctx, err) diff --git a/builder/store/database/mirror.go b/builder/store/database/mirror.go index d9ccd448..2d266d02 100644 --- a/builder/store/database/mirror.go +++ b/builder/store/database/mirror.go @@ -3,6 +3,7 @@ package database import ( "context" "fmt" + "strings" "time" "github.com/uptrace/bun" @@ -31,7 +32,8 @@ type MirrorStore interface { Finished(ctx context.Context) ([]Mirror, error) ToSyncRepo(ctx context.Context) ([]Mirror, error) ToSyncLfs(ctx context.Context) ([]Mirror, error) - IndexWithPagination(ctx context.Context, per, page int) (mirrors []Mirror, count int, err error) + IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error) + StatusCount(ctx context.Context) ([]MirrorStatusCount, error) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error } @@ -76,6 +78,11 @@ type Mirror struct { times } +type MirrorStatusCount struct { + Status types.MirrorTaskStatus `bun:"status"` + Count int `bun:"count"` +} + func (s *mirrorStoreImpl) IsExist(ctx context.Context, repoID int64) (exists bool, err error) { var mirror Mirror exists, err = s.db.Operator.Core. @@ -90,7 +97,8 @@ func (s *mirrorStoreImpl) IsRepoExist(ctx context.Context, repoType types.Reposi exists, err = s.db.Operator.Core. NewSelect(). Model(&repo). - Where("git_path=?", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)). + Where("path=?", fmt.Sprintf("%s/%s", namespace, name)). + Where("repository_type=?", repoType). Exists(ctx) return } @@ -273,7 +281,13 @@ func (s *mirrorStoreImpl) ToSyncRepo(ctx context.Context) ([]Mirror, error) { var mirrors []Mirror err := s.db.Operator.Core.NewSelect(). Model(&mirrors). - Where("next_execution_timestamp < ? or status in (?,?,?)", time.Now(), types.MirrorIncomplete, types.MirrorFailed, types.MirrorWaiting). + Where( + "next_execution_timestamp < ? or status in (?,?,?,?)", + time.Now(), + types.MirrorIncomplete, + types.MirrorFailed, + types.MirrorWaiting, + types.MirrorRunning). Scan(ctx) if err != nil { return nil, err @@ -293,11 +307,17 @@ func (s *mirrorStoreImpl) ToSyncLfs(ctx context.Context) ([]Mirror, error) { return mirrors, nil } -func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int) (mirrors []Mirror, count int, err error) { +func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error) { q := s.db.Operator.Core.NewSelect(). Model(&mirrors). Relation("Repository"). Relation("MirrorSource") + if search != "" { + q = q.Where("LOWER(mirror.source_url) like ? or LOWER(mirror.local_repo_path) like ?", + fmt.Sprintf("%%%s%%", strings.ToLower(search)), + fmt.Sprintf("%%%s%%", strings.ToLower(search)), + ) + } count, err = q.Count(ctx) if err != nil { return @@ -313,6 +333,17 @@ func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int return } +func (s *mirrorStoreImpl) StatusCount(ctx context.Context) ([]MirrorStatusCount, error) { + var statusCounts []MirrorStatusCount + err := s.db.Operator.Core.NewSelect(). + Model((*Mirror)(nil)). + Column("status"). + ColumnExpr("COUNT(*) AS count"). + Group("status"). + Scan(ctx, &statusCounts) + return statusCounts, err +} + func (s *mirrorStoreImpl) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error { err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { _, err := tx.NewUpdate().Model(mirror).WherePK().Exec(ctx) diff --git a/builder/store/database/mirror_test.go b/builder/store/database/mirror_test.go index ab056121..6175ebc3 100644 --- a/builder/store/database/mirror_test.go +++ b/builder/store/database/mirror_test.go @@ -51,6 +51,7 @@ func TestMirrorStore_CRUD(t *testing.T) { RepositoryType: types.ModelRepo, GitPath: "models_ns/n", Name: "repo", + Path: "ns/n", } err = db.Core.NewInsert().Model(repo).Scan(ctx, repo) require.Nil(t, err) @@ -194,7 +195,7 @@ func TestMirrorStore_ToSync(t *testing.T) { for _, m := range ms { names = append(names, m.Interval) } - require.ElementsMatch(t, []string{"m1", "m3", "m6", "m7"}, names) + require.ElementsMatch(t, []string{"m1", "m3", "m5", "m6", "m7"}, names) ms, err = store.ToSyncLfs(ctx) require.Nil(t, err) @@ -222,7 +223,7 @@ func TestMirrorStore_IndexWithPagination(t *testing.T) { require.Nil(t, err) } - ms, count, err := store.IndexWithPagination(ctx, 10, 1) + ms, count, err := store.IndexWithPagination(ctx, 10, 1, "foo") require.Nil(t, err) names := []string{} for _, m := range ms { @@ -250,4 +251,12 @@ func TestMirrorStore_StatusCount(t *testing.T) { require.Nil(t, err) } + cs, err := store.StatusCount(ctx) + require.Nil(t, err) + require.Equal(t, 2, len(cs)) + require.ElementsMatch(t, []database.MirrorStatusCount{ + {types.MirrorFailed, 2}, + {types.MirrorFinished, 1}, + }, cs) + } diff --git a/common/types/mirror.go b/common/types/mirror.go index 294bed0c..d079597c 100644 --- a/common/types/mirror.go +++ b/common/types/mirror.go @@ -135,3 +135,8 @@ type Mirror struct { type MirrorSource struct { SourceName string `json:"source_name"` } + +type MirrorStatusCount struct { + Status MirrorTaskStatus + Count int +} diff --git a/component/mirror.go b/component/mirror.go index 050ae92e..67aa642b 100644 --- a/component/mirror.go +++ b/component/mirror.go @@ -48,7 +48,8 @@ type MirrorComponent interface { CreateMirrorRepo(ctx context.Context, req types.CreateMirrorRepoReq) (*database.Mirror, error) CheckMirrorProgress(ctx context.Context) error Repos(ctx context.Context, currentUser string, per, page int) ([]types.MirrorRepo, int, error) - Index(ctx context.Context, currentUser string, per, page int) ([]types.Mirror, int, error) + Index(ctx context.Context, currentUser string, per, page int, search string) ([]types.Mirror, int, error) + Statistics(ctx context.Context, currentUser string) ([]types.MirrorStatusCount, error) } func NewMirrorComponent(config *config.Config) (MirrorComponent, error) { @@ -66,7 +67,8 @@ func NewMirrorComponent(config *config.Config) (MirrorComponent, error) { if err != nil { return nil, fmt.Errorf("failed to get priority queue: %v", err) } - c.repoComp, err = NewRepoComponent(config) + + c.repoComp, err = NewRepoComponentImpl(config) if err != nil { return nil, fmt.Errorf("fail to create repo component,error:%w", err) } @@ -246,6 +248,7 @@ func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.Cr mirror.LocalRepoPath = fmt.Sprintf("%s_%s_%s_%s", mirrorSource.SourceName, req.RepoType, req.SourceNamespace, req.SourceName) mirror.SourceRepoPath = fmt.Sprintf("%s/%s", req.SourceNamespace, req.SourceName) mirror.Priority = types.HighMirrorPriority + var taskId int64 if c.config.GitServer.Type == types.GitServerTypeGitea { taskId, err = c.mirrorServer.CreateMirrorRepo(ctx, mirrorserver.CreateMirrorRepoReq{ @@ -262,14 +265,12 @@ func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.Cr return nil, fmt.Errorf("failed to create push mirror in mirror server: %v", err) } } - mirror.MirrorTaskID = taskId reqMirror, err := c.mirrorStore.Create(ctx, &mirror) if err != nil { return nil, fmt.Errorf("failed to create mirror") } - if c.config.GitServer.Type == types.GitServerTypeGitaly { c.mq.PushRepoMirror(&queue.MirrorTask{ MirrorID: reqMirror.ID, @@ -283,6 +284,7 @@ func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.Cr } return reqMirror, nil + } func (m *mirrorComponentImpl) mapNamespaceAndName(sourceNamespace string) string { @@ -620,7 +622,7 @@ func (c *mirrorComponentImpl) Repos(ctx context.Context, currentUser string, per return mirrorRepos, total, nil } -func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per, page int) ([]types.Mirror, int, error) { +func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per, page int, search string) ([]types.Mirror, int, error) { var mirrorsResp []types.Mirror user, err := c.userStore.FindByUsername(ctx, currentUser) if err != nil { @@ -629,28 +631,54 @@ func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per if !user.CanAdmin() { return nil, 0, errors.New("user does not have admin permission") } - mirrors, total, err := c.mirrorStore.IndexWithPagination(ctx, per, page) + mirrors, total, err := c.mirrorStore.IndexWithPagination(ctx, per, page, search) if err != nil { return nil, 0, fmt.Errorf("failed to get mirror mirrors: %v", err) } for _, mirror := range mirrors { - mirrorsResp = append(mirrorsResp, types.Mirror{ - SourceUrl: mirror.SourceUrl, - MirrorSource: types.MirrorSource{ - SourceName: mirror.MirrorSource.SourceName, - }, - Username: mirror.Username, - AccessToken: mirror.AccessToken, - PushUrl: mirror.PushUrl, - PushUsername: mirror.PushUsername, - PushAccessToken: mirror.PushAccessToken, - LastUpdatedAt: mirror.LastUpdatedAt, - SourceRepoPath: mirror.SourceRepoPath, - LocalRepoPath: fmt.Sprintf("%ss/%s", mirror.Repository.RepositoryType, mirror.Repository.Path), - LastMessage: mirror.LastMessage, - Status: mirror.Status, - Progress: mirror.Progress, - }) + if mirror.Repository != nil { + mirrorsResp = append(mirrorsResp, types.Mirror{ + SourceUrl: mirror.SourceUrl, + MirrorSource: types.MirrorSource{ + SourceName: mirror.MirrorSource.SourceName, + }, + Username: mirror.Username, + AccessToken: mirror.AccessToken, + PushUrl: mirror.PushUrl, + PushUsername: mirror.PushUsername, + PushAccessToken: mirror.PushAccessToken, + LastUpdatedAt: mirror.LastUpdatedAt, + SourceRepoPath: mirror.SourceRepoPath, + LocalRepoPath: fmt.Sprintf("%ss/%s", mirror.Repository.RepositoryType, mirror.Repository.Path), + LastMessage: mirror.LastMessage, + Status: mirror.Status, + Progress: mirror.Progress, + }) + } } return mirrorsResp, total, nil } + +func (c *mirrorComponentImpl) Statistics(ctx context.Context, currentUser string) ([]types.MirrorStatusCount, error) { + var scs []types.MirrorStatusCount + user, err := c.userStore.FindByUsername(ctx, currentUser) + if err != nil { + return nil, errors.New("user does not exist") + } + if !user.CanAdmin() { + return nil, errors.New("user does not have admin permission") + } + statusCounts, err := c.mirrorStore.StatusCount(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get mirror statistics: %v", err) + } + + for _, statusCount := range statusCounts { + scs = append(scs, types.MirrorStatusCount{ + Status: statusCount.Status, + Count: statusCount.Count, + }) + } + + return scs, nil +} diff --git a/component/mirror_test.go b/component/mirror_test.go new file mode 100644 index 00000000..375002c3 --- /dev/null +++ b/component/mirror_test.go @@ -0,0 +1,355 @@ +package component + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "opencsg.com/csghub-server/builder/git/gitserver" + "opencsg.com/csghub-server/builder/git/mirrorserver" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/mirror/queue" +) + +func TestMirrorComponent_CreatePushMirrorForFinishedMirrorTask(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.MirrorMock().EXPECT().NoPushMirror(ctx).Return([]database.Mirror{ + {MirrorTaskID: 1}, + {MirrorTaskID: 2, LocalRepoPath: "foo"}, + }, nil) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(1)).Return( + &mirrorserver.MirrorTaskInfo{}, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(2)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusFinished, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().CreatePushMirror(ctx, mirrorserver.CreatePushMirrorReq{ + Name: "foo", + Interval: "8h", + }).Return(nil) + mc.mocks.stores.MirrorMock().EXPECT().Update(ctx, &database.Mirror{ + MirrorTaskID: 2, LocalRepoPath: "foo", PushMirrorCreated: true, + }).Return(nil) + + err := mc.CreatePushMirrorForFinishedMirrorTask(ctx) + require.Nil(t, err) +} + +func TestMirrorComponent_CreateMirrorRepo(t *testing.T) { + + cases := []struct { + repoType types.RepositoryType + gitea bool + }{ + {types.ModelRepo, false}, + {types.DatasetRepo, false}, + {types.CodeRepo, false}, + {types.CodeRepo, true}, + } + + for _, c := range cases { + t.Run(fmt.Sprintf("%+v", c), func(t *testing.T) { + + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + req := types.CreateMirrorRepoReq{ + SourceNamespace: "sns", + SourceName: "sn", + RepoType: c.repoType, + CurrentUser: "user", + } + + if c.gitea { + mc.config.GitServer.Type = types.GitServerTypeGitea + } else { + mc.config.GitServer.Type = types.GitServerTypeGitaly + } + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + repo := &database.Repository{} + mc.mocks.stores.RepoMock().EXPECT().FindByPath( + ctx, req.RepoType, "AIWizards", "sn", + ).Return( + repo, nil, + ) + mc.mocks.stores.RepoMock().EXPECT().FindByPath( + ctx, req.RepoType, "AIWizards", "sns_sn", + ).Return( + nil, sql.ErrNoRows, + ) + mc.mocks.stores.NamespaceMock().EXPECT().FindByPath( + ctx, "AIWizards", + ).Return(database.Namespace{ + User: database.User{Username: "user"}, + }, nil) + mc.mocks.components.repo.EXPECT().CreateRepo(ctx, types.CreateRepoReq{ + Username: "user", + Namespace: "AIWizards", + Name: "sns_sn", + Nickname: "sns_sn", + Description: req.Description, + Private: true, + License: req.License, + DefaultBranch: req.DefaultBranch, + RepoType: req.RepoType, + }).Return(&gitserver.CreateRepoResp{}, &database.Repository{}, nil) + switch req.RepoType { + case types.ModelRepo: + mc.mocks.stores.ModelMock().EXPECT().Create(ctx, database.Model{ + Repository: repo, + RepositoryID: repo.ID, + }).Return(nil, nil) + case types.DatasetRepo: + mc.mocks.stores.DatasetMock().EXPECT().Create(ctx, database.Dataset{ + Repository: repo, + RepositoryID: repo.ID, + }).Return(nil, nil) + case types.CodeRepo: + mc.mocks.stores.CodeMock().EXPECT().Create(ctx, database.Code{ + Repository: repo, + RepositoryID: repo.ID, + }).Return(nil, nil) + } + mc.mocks.stores.GitServerAccessTokenMock().EXPECT().FindByType(ctx, "git").Return( + []database.GitServerAccessToken{ + {}, + }, nil, + ) + mc.mocks.stores.MirrorSourceMock().EXPECT().Get(ctx, int64(0)).Return( + &database.MirrorSource{}, nil, + ) + if c.gitea { + mc.mocks.mirrorServer.EXPECT().CreateMirrorRepo(ctx, mirrorserver.CreateMirrorRepoReq{ + Name: "_code_sns_sn", + Namespace: "root", + Private: false, + SyncLfs: req.SyncLfs, + }).Return(123, nil) + } + reqMirror := &database.Mirror{ + ID: 1, + Priority: types.HighMirrorPriority, + } + localRepoPath := "" + switch req.RepoType { + case types.ModelRepo: + localRepoPath = "_model_sns_sn" + case types.DatasetRepo: + localRepoPath = "_dataset_sns_sn" + case types.CodeRepo: + localRepoPath = "_code_sns_sn" + } + + cm := &database.Mirror{ + Username: "sns", + PushUsername: "root", + SourceRepoPath: "sns/sn", + LocalRepoPath: localRepoPath, + Priority: types.HighMirrorPriority, + Repository: &database.Repository{}, + } + if c.gitea { + cm.MirrorTaskID = 123 + } + mc.mocks.stores.MirrorMock().EXPECT().Create(ctx, cm).Return( + reqMirror, nil, + ) + if !c.gitea { + mc.mocks.mirrorQueue.EXPECT().PushRepoMirror(&queue.MirrorTask{ + MirrorID: reqMirror.ID, + Priority: queue.PriorityMap[reqMirror.Priority], + }) + mc.mocks.stores.MirrorMock().EXPECT().Update(ctx, reqMirror).Return(nil) + } + + m, err := mc.CreateMirrorRepo(ctx, req) + require.Nil(t, err) + require.Equal(t, reqMirror, m) + + }) + } + +} + +func TestMirrorComponent_CheckMirrorProgress(t *testing.T) { + + for _, saas := range []bool{false, true} { + t.Run(fmt.Sprintf("saas %v", saas), func(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + mc.saas = saas + + mirrors := []database.Mirror{ + { + ID: 1, MirrorTaskID: 11, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + { + ID: 2, MirrorTaskID: 12, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + { + ID: 3, MirrorTaskID: 13, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + { + ID: 4, MirrorTaskID: 14, + Repository: &database.Repository{ + ID: 111, Path: "foo/bar", RepositoryType: types.ModelRepo, + }, + }, + } + mc.mocks.stores.MirrorMock().EXPECT().Unfinished(ctx).Return(mirrors, nil) + + if saas { + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(11)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusQueued, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(12)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusRunning, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(13)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusFailed, + }, nil, + ) + mc.mocks.mirrorServer.EXPECT().GetMirrorTaskInfo(ctx, int64(14)).Return( + &mirrorserver.MirrorTaskInfo{ + Status: mirrorserver.TaskStatusFinished, + }, nil, + ) + } else { + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(11)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusQueued, + }, nil, + ) + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(12)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusRunning, + }, nil, + ) + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(13)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusFailed, + }, nil, + ) + mc.mocks.gitServer.EXPECT().GetMirrorTaskInfo(ctx, int64(14)).Return( + &gitserver.MirrorTaskInfo{ + Status: gitserver.TaskStatusFinished, + }, nil, + ) + } + mirrors[0].Status = types.MirrorWaiting + mirrors[1].Status = types.MirrorRunning + mirrors[1].Progress = 100 + mirrors[2].Status = types.MirrorFailed + mirrors[3].Status = types.MirrorFinished + mirrors[3].Progress = 100 + mc.mocks.gitServer.EXPECT().GetRepo(ctx, gitserver.GetRepoReq{ + Namespace: "foo", + Name: "bar", + RepoType: types.ModelRepo, + }).Return(&gitserver.CreateRepoResp{}, nil) + for _, m := range mirrors { + m.Repository.SyncStatus = mirrorStatusAndRepoSyncStatusMapping[m.Status] + mv := m + mc.mocks.stores.MirrorMock().EXPECT().Update(ctx, &mv).Return(nil).Once() + mc.mocks.stores.RepoMock().EXPECT().UpdateRepo( + ctx, database.Repository{ + ID: 111, + Path: "foo/bar", + RepositoryType: types.ModelRepo, + SyncStatus: mirrorStatusAndRepoSyncStatusMapping[mv.Status], + }, + ).Return(nil, nil).Once() + } + mc.mocks.gitServer.EXPECT().GetRepoFileTree( + mock.Anything, gitserver.GetRepoInfoByPathReq{ + Namespace: "foo", Name: "bar", RepoType: "model"}, + ).Return([]*types.File{{Name: "foo.go"}}, nil) + + err := mc.CheckMirrorProgress(ctx) + require.Nil(t, err) + }) + } + +} + +func TestMirrorComponent_Repos(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + mc.mocks.stores.RepoMock().EXPECT().WithMirror(ctx, 10, 1).Return([]database.Repository{ + {Path: "foo", SyncStatus: types.SyncStatusCompleted, RepositoryType: types.ModelRepo}, + }, 100, nil) + + data, total, err := mc.Repos(ctx, "user", 10, 1) + require.Nil(t, err) + require.Equal(t, 100, total) + require.Equal(t, []types.MirrorRepo{ + {Path: "foo", SyncStatus: types.SyncStatusCompleted, RepoType: types.ModelRepo}, + }, data) +} + +func TestMirrorComponent_Index(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + mc.mocks.stores.MirrorMock().EXPECT().IndexWithPagination(ctx, 10, 1, "foo").Return( + []database.Mirror{{Username: "user", LastMessage: "msg", Repository: &database.Repository{}}}, 100, nil, + ) + + data, total, err := mc.Index(ctx, "user", 10, 1, "foo") + require.Nil(t, err) + require.Equal(t, 100, total) + require.Equal(t, []types.Mirror{ + {Username: "user", LastMessage: "msg", LocalRepoPath: "s/"}, + }, data) +} + +func TestMirrorComponent_Statistic(t *testing.T) { + ctx := context.TODO() + mc := initializeTestMirrorComponent(ctx, t) + + mc.mocks.stores.UserMock().EXPECT().FindByUsername(ctx, "user").Return(database.User{ + RoleMask: "admin", + }, nil) + mc.mocks.stores.MirrorMock().EXPECT().StatusCount(ctx).Return([]database.MirrorStatusCount{ + {Status: types.MirrorFinished, Count: 100}, + }, nil) + + s, err := mc.Statistics(ctx, "user") + require.Nil(t, err) + require.Equal(t, []types.MirrorStatusCount{ + {Status: types.MirrorFinished, Count: 100}, + }, s) + +}