Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync mirror/list/dataset/code component with enterprise #213

Merged
merged 4 commits into from
Dec 18, 2024
Merged

Sync mirror/list/dataset/code component with enterprise #213

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2e7daac
sync code component
Yiling-J Dec 18, 2024
2d483be
sync dataset component
Yiling-J Dec 18, 2024
c3c320b
sync list component
Yiling-J Dec 18, 2024
3445076
sync mirror component
Yiling-J Dec 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 75 additions & 16 deletions _mocks/opencsg.com/csghub-server/builder/store/database/mock_MirrorStore.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion api/handler/mirror.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,8 @@ func (h *MirrorHandler) Index(ctx *gin.Context) {
httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
return
}
repos, total, err := h.mc.Index(ctx, currentUser, per, page)
search := ctx.Query("search")
Yiling-J marked this conversation as resolved.
Show resolved Hide resolved
repos, total, err := h.mc.Index(ctx, currentUser, per, page, search)
if err != nil {
slog.Error("failed to get mirror repos", slog.Any("error", err))
httpbase.ServerError(ctx, err)
Expand Down
39 changes: 35 additions & 4 deletions builder/store/database/mirror.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package database
import (
"context"
"fmt"
"strings"
Yiling-J marked this conversation as resolved.
Show resolved Hide resolved
"time"

"github.com/uptrace/bun"
Expand Down Expand Up @@ -31,7 +32,8 @@ type MirrorStore interface {
Finished(ctx context.Context) ([]Mirror, error)
ToSyncRepo(ctx context.Context) ([]Mirror, error)
ToSyncLfs(ctx context.Context) ([]Mirror, error)
IndexWithPagination(ctx context.Context, per, page int) (mirrors []Mirror, count int, err error)
IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error)
StatusCount(ctx context.Context) ([]MirrorStatusCount, error)
UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error
}

Expand Down Expand Up @@ -76,6 +78,11 @@ type Mirror struct {
times
}

type MirrorStatusCount struct {
Status types.MirrorTaskStatus `bun:"status"`
Count int `bun:"count"`
}

func (s *mirrorStoreImpl) IsExist(ctx context.Context, repoID int64) (exists bool, err error) {
var mirror Mirror
exists, err = s.db.Operator.Core.
Expand All @@ -90,7 +97,8 @@ func (s *mirrorStoreImpl) IsRepoExist(ctx context.Context, repoType types.Reposi
exists, err = s.db.Operator.Core.
NewSelect().
Model(&repo).
Where("git_path=?", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)).
Where("path=?", fmt.Sprintf("%s/%s", namespace, name)).
Where("repository_type=?", repoType).
Exists(ctx)
return
}
Expand Down Expand Up @@ -273,7 +281,13 @@ func (s *mirrorStoreImpl) ToSyncRepo(ctx context.Context) ([]Mirror, error) {
var mirrors []Mirror
err := s.db.Operator.Core.NewSelect().
Model(&mirrors).
Where("next_execution_timestamp < ? or status in (?,?,?)", time.Now(), types.MirrorIncomplete, types.MirrorFailed, types.MirrorWaiting).
Where(
"next_execution_timestamp < ? or status in (?,?,?,?)",
time.Now(),
types.MirrorIncomplete,
types.MirrorFailed,
types.MirrorWaiting,
types.MirrorRunning).
Yiling-J marked this conversation as resolved.
Show resolved Hide resolved
Scan(ctx)
if err != nil {
return nil, err
Expand All @@ -293,11 +307,17 @@ func (s *mirrorStoreImpl) ToSyncLfs(ctx context.Context) ([]Mirror, error) {
return mirrors, nil
}

func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int) (mirrors []Mirror, count int, err error) {
func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error) {
q := s.db.Operator.Core.NewSelect().
Model(&mirrors).
Relation("Repository").
Relation("MirrorSource")
if search != "" {
q = q.Where("LOWER(mirror.source_url) like ? or LOWER(mirror.local_repo_path) like ?",
fmt.Sprintf("%%%s%%", strings.ToLower(search)),
fmt.Sprintf("%%%s%%", strings.ToLower(search)),
)
}
count, err = q.Count(ctx)
if err != nil {
return
Expand All @@ -313,6 +333,17 @@ func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int
return
}

func (s *mirrorStoreImpl) StatusCount(ctx context.Context) ([]MirrorStatusCount, error) {
var statusCounts []MirrorStatusCount
err := s.db.Operator.Core.NewSelect().
Model((*Mirror)(nil)).
Column("status").
ColumnExpr("COUNT(*) AS count").
Group("status").
Scan(ctx, &statusCounts)
return statusCounts, err
}

func (s *mirrorStoreImpl) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error {
err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
_, err := tx.NewUpdate().Model(mirror).WherePK().Exec(ctx)
Expand Down
13 changes: 11 additions & 2 deletions builder/store/database/mirror_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func TestMirrorStore_CRUD(t *testing.T) {
RepositoryType: types.ModelRepo,
GitPath: "models_ns/n",
Name: "repo",
Path: "ns/n",
}
err = db.Core.NewInsert().Model(repo).Scan(ctx, repo)
require.Nil(t, err)
Expand Down Expand Up @@ -194,7 +195,7 @@ func TestMirrorStore_ToSync(t *testing.T) {
for _, m := range ms {
names = append(names, m.Interval)
}
require.ElementsMatch(t, []string{"m1", "m3", "m6", "m7"}, names)
require.ElementsMatch(t, []string{"m1", "m3", "m5", "m6", "m7"}, names)
Yiling-J marked this conversation as resolved.
Show resolved Hide resolved

ms, err = store.ToSyncLfs(ctx)
require.Nil(t, err)
Expand Down Expand Up @@ -222,7 +223,7 @@ func TestMirrorStore_IndexWithPagination(t *testing.T) {
require.Nil(t, err)
}

ms, count, err := store.IndexWithPagination(ctx, 10, 1)
ms, count, err := store.IndexWithPagination(ctx, 10, 1, "foo")
require.Nil(t, err)
names := []string{}
for _, m := range ms {
Expand Down Expand Up @@ -250,4 +251,12 @@ func TestMirrorStore_StatusCount(t *testing.T) {
require.Nil(t, err)
}

cs, err := store.StatusCount(ctx)
require.Nil(t, err)
require.Equal(t, 2, len(cs))
require.ElementsMatch(t, []database.MirrorStatusCount{
{types.MirrorFailed, 2},
{types.MirrorFinished, 1},
}, cs)

}
45 changes: 23 additions & 22 deletions common/types/code.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,27 @@ type UpdateCodeReq struct {
}

type Code struct {
ID int64 `json:"id"`
Name string `json:"name"`
Nickname string `json:"nickname"`
Description string `json:"description"`
Likes int64 `json:"likes"`
Downloads int64 `json:"downloads"`
Path string `json:"path"`
RepositoryID int64 `json:"repository_id"`
Repository Repository `json:"repository"`
Private bool `json:"private"`
User User `json:"user"`
Tags []RepoTag `json:"tags"`
DefaultBranch string `json:"default_branch"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserLikes bool `json:"user_likes"`
Source RepositorySource `json:"source"`
SyncStatus RepositorySyncStatus `json:"sync_status"`
License string `json:"license"`
CanWrite bool `json:"can_write"`
CanManage bool `json:"can_manage"`
Namespace *Namespace `json:"namespace"`
ID int64 `json:"id"`
Name string `json:"name"`
Nickname string `json:"nickname"`
Description string `json:"description"`
Likes int64 `json:"likes"`
Downloads int64 `json:"downloads"`
Path string `json:"path"`
RepositoryID int64 `json:"repository_id"`
Repository Repository `json:"repository"`
Private bool `json:"private"`
User User `json:"user"`
Tags []RepoTag `json:"tags"`
DefaultBranch string `json:"default_branch"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserLikes bool `json:"user_likes"`
Source RepositorySource `json:"source"`
SyncStatus RepositorySyncStatus `json:"sync_status"`
License string `json:"license"`
CanWrite bool `json:"can_write"`
CanManage bool `json:"can_manage"`
Namespace *Namespace `json:"namespace"`
SensitiveCheckStatus string `json:"sensitive_check_status"`
}
49 changes: 25 additions & 24 deletions common/types/dataset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,29 @@ type UpdateDatasetReq struct {
}

type Dataset struct {
ID int64 `json:"id,omitempty"`
Name string `json:"name"`
Nickname string `json:"nickname"`
Description string `json:"description"`
Likes int64 `json:"likes"`
Downloads int64 `json:"downloads"`
Path string `json:"path"`
RepositoryID int64 `json:"repository_id"`
Repository Repository `json:"repository"`
Private bool `json:"private"`
User User `json:"user"`
Tags []RepoTag `json:"tags"`
Readme string `json:"readme"`
DefaultBranch string `json:"default_branch"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserLikes bool `json:"user_likes"`
Source RepositorySource `json:"source"`
SyncStatus RepositorySyncStatus `json:"sync_status"`
License string `json:"license"`
CanWrite bool `json:"can_write"`
CanManage bool `json:"can_manage"`
Namespace *Namespace `json:"namespace"`
MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"`
ID int64 `json:"id,omitempty"`
Name string `json:"name"`
Nickname string `json:"nickname"`
Description string `json:"description"`
Likes int64 `json:"likes"`
Downloads int64 `json:"downloads"`
Path string `json:"path"`
RepositoryID int64 `json:"repository_id"`
Repository Repository `json:"repository"`
Private bool `json:"private"`
User User `json:"user"`
Tags []RepoTag `json:"tags"`
Readme string `json:"readme"`
DefaultBranch string `json:"default_branch"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
UserLikes bool `json:"user_likes"`
Source RepositorySource `json:"source"`
SyncStatus RepositorySyncStatus `json:"sync_status"`
License string `json:"license"`
CanWrite bool `json:"can_write"`
CanManage bool `json:"can_manage"`
Namespace *Namespace `json:"namespace"`
SensitiveCheckStatus string `json:"sensitive_check_status"`
MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"`
}
5 changes: 5 additions & 0 deletions common/types/mirror.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,8 @@ type Mirror struct {
type MirrorSource struct {
SourceName string `json:"source_name"`
}

type MirrorStatusCount struct {
Status MirrorTaskStatus
Count int
}
Loading
Loading