From 5d7b63cde29b487ce8cd580870fb315a43fda8d5 Mon Sep 17 00:00:00 2001 From: James <370036720@qq.com> Date: Fri, 6 Dec 2024 12:34:18 +0800 Subject: [PATCH] Support model evaluation (#197) * Add missing database tests Support argo workflow system Support evaluation datasets support opencompass * Evaluation bug fix * fix bugs * support master hosts support harness evaluation * fix * fix * fix bug * add readme * update doc * update lint and test * fix ut * update repo * rollback test * update * update readme --------- Co-authored-by: yiling Co-authored-by: James --- .gitignore | 2 + .../builder/deploy/mock_Deployer.go | 226 ++++++++++ .../store/database/mock_ArgoWorkFlowStore.go | 384 +++++++++++++++++ .../store/database/mock_TagRuleStore.go | 99 +++++ .../builder/store/database/mock_TagStore.go | 61 +++ .../component/mock_TagComponent.go | 60 +++ api/handler/evaluation.go | 147 +++++++ api/handler/repo.go | 2 +- api/handler/tag.go | 7 +- api/handler/user.go | 46 +++ api/router/api.go | 28 +- builder/deploy/cluster/cluster_manager.go | 7 + builder/deploy/common/deploy_config.go | 3 +- builder/deploy/deployer.go | 69 +++- builder/deploy/imagerunner/local_runner.go | 17 + builder/deploy/imagerunner/remote_runner.go | 66 +++ builder/deploy/imagerunner/runner.go | 5 + builder/store/database/argo_workflow.go | 121 ++++++ .../20241105124716_create_table_workflows.go | 51 +++ .../20241111095821_create_tag_rules.go | 35 ++ .../20241111095847_init_tag_rule.down.sql | 9 + .../20241111095847_init_tag_rule.up.sql | 112 +++++ ...095440_update_argo_workflow_table.down.sql | 10 + ...17095440_update_argo_workflow_table.up.sql | 18 + ...0241126071538_add_lm_harness_data.down.sql | 10 + .../20241126071538_add_lm_harness_data.up.sql | 160 ++++++++ builder/store/database/repository.go | 11 +- builder/store/database/repository_test.go | 2 +- .../store/database/seeds/tag_categories.yml | 26 +- builder/store/database/tag.go | 33 +- builder/store/database/tag_rule.go | 52 +++ builder/store/database/tag_rule_test.go | 34 ++ cmd/csghub-server/cmd/deploy/runner.go | 12 +- cmd/csghub-server/cmd/start/server.go | 1 + common/config/config.go | 14 + common/types/accounting.go | 1 + common/types/argo_workflow.go | 157 +++++++ common/types/mirror.go | 7 +- common/types/model.go | 29 +- common/types/tag.go | 11 + common/types/user.go | 1 + component/callback/git_callback.go | 78 +++- component/evaluation.go | 191 +++++++++ component/evaluation_test.go | 198 +++++++++ component/mirror.go | 1 + component/model.go | 22 +- component/model_test.go | 18 +- component/runtime_architecture.go | 1 + component/tag.go | 8 +- component/user.go | 21 + .../Dockerfile.lm-evaluation-harness | 17 + docker/evaluation/Dockerfile.opencompass | 15 + docker/evaluation/README.md | 66 +++ .../lm-evaluation-harness/download.py | 27 ++ .../lm-evaluation-harness/get_task.py | 112 +++++ .../evaluation/lm-evaluation-harness/start.sh | 123 ++++++ .../lm-evaluation-harness/upload_files.py | 222 ++++++++++ .../evaluation/opencompass/get_answer_mode.py | 30 ++ docker/evaluation/opencompass/start.sh | 91 +++++ docker/evaluation/opencompass/upload_files.py | 201 +++++++++ docker/inference/Dockerfile.tgi | 10 +- docker/inference/supervisord.conf | 1 + docs/docs.go | 381 ++++++++++++++++- docs/swagger.json | 381 ++++++++++++++++- docs/swagger.yaml | 248 +++++++++++ go.mod | 50 +-- go.sum | 98 +++-- .../component/service.go | 4 +- runner/component/workflow.go | 385 ++++++++++++++++++ .../k8s.go => runner/handler/service.go | 63 ++- runner/handler/workflow.go | 153 +++++++ runner/router/api.go | 65 +++ servicerunner/router/api.go | 45 -- 73 files changed, 5251 insertions(+), 221 deletions(-) create mode 100644 _mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go create mode 100644 _mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go create mode 100644 api/handler/evaluation.go create mode 100644 builder/store/database/argo_workflow.go create mode 100644 builder/store/database/migrations/20241105124716_create_table_workflows.go create mode 100644 builder/store/database/migrations/20241111095821_create_tag_rules.go create mode 100644 builder/store/database/migrations/20241111095847_init_tag_rule.down.sql create mode 100644 builder/store/database/migrations/20241111095847_init_tag_rule.up.sql create mode 100644 builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql create mode 100644 builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql create mode 100644 builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql create mode 100644 builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql create mode 100644 builder/store/database/tag_rule.go create mode 100644 builder/store/database/tag_rule_test.go create mode 100644 common/types/argo_workflow.go create mode 100644 component/evaluation.go create mode 100644 component/evaluation_test.go create mode 100644 docker/evaluation/Dockerfile.lm-evaluation-harness create mode 100644 docker/evaluation/Dockerfile.opencompass create mode 100644 docker/evaluation/README.md create mode 100644 docker/evaluation/lm-evaluation-harness/download.py create mode 100644 docker/evaluation/lm-evaluation-harness/get_task.py create mode 100644 docker/evaluation/lm-evaluation-harness/start.sh create mode 100644 docker/evaluation/lm-evaluation-harness/upload_files.py create mode 100644 docker/evaluation/opencompass/get_answer_mode.py create mode 100644 docker/evaluation/opencompass/start.sh create mode 100644 docker/evaluation/opencompass/upload_files.py rename {servicerunner => runner}/component/service.go (97%) create mode 100644 runner/component/workflow.go rename servicerunner/handler/k8s.go => runner/handler/service.go (93%) create mode 100644 runner/handler/workflow.go create mode 100644 runner/router/api.go delete mode 100644 servicerunner/router/api.go diff --git a/.gitignore b/.gitignore index 45bfe3ab..47bd7279 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ local.toml .data /data */*/_api_test/ +redis/dump.rdb +run_server_local.sh diff --git a/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go b/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go index 2f067af4..954cb181 100644 --- a/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go +++ b/_mocks/opencsg.com/csghub-server/builder/deploy/mock_Deployer.go @@ -84,6 +84,53 @@ func (_c *MockDeployer_CheckResourceAvailable_Call) RunAndReturn(run func(contex return _c } +// DeleteEvaluation provides a mock function with given fields: ctx, req +func (_m *MockDeployer) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for DeleteEvaluation") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, types.ArgoWorkFlowDeleteReq) error); ok { + r0 = rf(ctx, req) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockDeployer_DeleteEvaluation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteEvaluation' +type MockDeployer_DeleteEvaluation_Call struct { + *mock.Call +} + +// DeleteEvaluation is a helper method to define mock.On call +// - ctx context.Context +// - req types.ArgoWorkFlowDeleteReq +func (_e *MockDeployer_Expecter) DeleteEvaluation(ctx interface{}, req interface{}) *MockDeployer_DeleteEvaluation_Call { + return &MockDeployer_DeleteEvaluation_Call{Call: _e.mock.On("DeleteEvaluation", ctx, req)} +} + +func (_c *MockDeployer_DeleteEvaluation_Call) Run(run func(ctx context.Context, req types.ArgoWorkFlowDeleteReq)) *MockDeployer_DeleteEvaluation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(types.ArgoWorkFlowDeleteReq)) + }) + return _c +} + +func (_c *MockDeployer_DeleteEvaluation_Call) Return(_a0 error) *MockDeployer_DeleteEvaluation_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockDeployer_DeleteEvaluation_Call) RunAndReturn(run func(context.Context, types.ArgoWorkFlowDeleteReq) error) *MockDeployer_DeleteEvaluation_Call { + _c.Call.Return(run) + return _c +} + // Deploy provides a mock function with given fields: ctx, dr func (_m *MockDeployer) Deploy(ctx context.Context, dr types.DeployRepo) (int64, error) { ret := _m.Called(ctx, dr) @@ -257,6 +304,65 @@ func (_c *MockDeployer_GetClusterById_Call) RunAndReturn(run func(context.Contex return _c } +// GetEvaluation provides a mock function with given fields: ctx, req +func (_m *MockDeployer) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for GetEvaluation") + } + + var r0 *types.ArgoWorkFlowRes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationGetReq) *types.ArgoWorkFlowRes); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.ArgoWorkFlowRes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, types.EvaluationGetReq) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployer_GetEvaluation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetEvaluation' +type MockDeployer_GetEvaluation_Call struct { + *mock.Call +} + +// GetEvaluation is a helper method to define mock.On call +// - ctx context.Context +// - req types.EvaluationGetReq +func (_e *MockDeployer_Expecter) GetEvaluation(ctx interface{}, req interface{}) *MockDeployer_GetEvaluation_Call { + return &MockDeployer_GetEvaluation_Call{Call: _e.mock.On("GetEvaluation", ctx, req)} +} + +func (_c *MockDeployer_GetEvaluation_Call) Run(run func(ctx context.Context, req types.EvaluationGetReq)) *MockDeployer_GetEvaluation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(types.EvaluationGetReq)) + }) + return _c +} + +func (_c *MockDeployer_GetEvaluation_Call) Return(_a0 *types.ArgoWorkFlowRes, _a1 error) *MockDeployer_GetEvaluation_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployer_GetEvaluation_Call) RunAndReturn(run func(context.Context, types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error)) *MockDeployer_GetEvaluation_Call { + _c.Call.Return(run) + return _c +} + // GetReplica provides a mock function with given fields: ctx, dr func (_m *MockDeployer) GetReplica(ctx context.Context, dr types.DeployRepo) (int, int, []types.Instance, error) { ret := _m.Called(ctx, dr) @@ -447,6 +553,67 @@ func (_c *MockDeployer_ListCluster_Call) RunAndReturn(run func(context.Context) return _c } +// ListEvaluations provides a mock function with given fields: _a0, _a1, _a2, _a3 +func (_m *MockDeployer) ListEvaluations(_a0 context.Context, _a1 string, _a2 int, _a3 int) (*types.ArgoWorkFlowListRes, error) { + ret := _m.Called(_a0, _a1, _a2, _a3) + + if len(ret) == 0 { + panic("no return value specified for ListEvaluations") + } + + var r0 *types.ArgoWorkFlowListRes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error)); ok { + return rf(_a0, _a1, _a2, _a3) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) *types.ArgoWorkFlowListRes); ok { + r0 = rf(_a0, _a1, _a2, _a3) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.ArgoWorkFlowListRes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int, int) error); ok { + r1 = rf(_a0, _a1, _a2, _a3) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployer_ListEvaluations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListEvaluations' +type MockDeployer_ListEvaluations_Call struct { + *mock.Call +} + +// ListEvaluations is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 string +// - _a2 int +// - _a3 int +func (_e *MockDeployer_Expecter) ListEvaluations(_a0 interface{}, _a1 interface{}, _a2 interface{}, _a3 interface{}) *MockDeployer_ListEvaluations_Call { + return &MockDeployer_ListEvaluations_Call{Call: _e.mock.On("ListEvaluations", _a0, _a1, _a2, _a3)} +} + +func (_c *MockDeployer_ListEvaluations_Call) Run(run func(_a0 context.Context, _a1 string, _a2 int, _a3 int)) *MockDeployer_ListEvaluations_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(int), args[3].(int)) + }) + return _c +} + +func (_c *MockDeployer_ListEvaluations_Call) Return(_a0 *types.ArgoWorkFlowListRes, _a1 error) *MockDeployer_ListEvaluations_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployer_ListEvaluations_Call) RunAndReturn(run func(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error)) *MockDeployer_ListEvaluations_Call { + _c.Call.Return(run) + return _c +} + // Logs provides a mock function with given fields: ctx, dr func (_m *MockDeployer) Logs(ctx context.Context, dr types.DeployRepo) (*deploy.MultiLogReader, error) { ret := _m.Called(ctx, dr) @@ -721,6 +888,65 @@ func (_c *MockDeployer_Stop_Call) RunAndReturn(run func(context.Context, types.D return _c } +// SubmitEvaluation provides a mock function with given fields: ctx, req +func (_m *MockDeployer) SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + ret := _m.Called(ctx, req) + + if len(ret) == 0 { + panic("no return value specified for SubmitEvaluation") + } + + var r0 *types.ArgoWorkFlowRes + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationReq) (*types.ArgoWorkFlowRes, error)); ok { + return rf(ctx, req) + } + if rf, ok := ret.Get(0).(func(context.Context, types.EvaluationReq) *types.ArgoWorkFlowRes); ok { + r0 = rf(ctx, req) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.ArgoWorkFlowRes) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, types.EvaluationReq) error); ok { + r1 = rf(ctx, req) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockDeployer_SubmitEvaluation_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SubmitEvaluation' +type MockDeployer_SubmitEvaluation_Call struct { + *mock.Call +} + +// SubmitEvaluation is a helper method to define mock.On call +// - ctx context.Context +// - req types.EvaluationReq +func (_e *MockDeployer_Expecter) SubmitEvaluation(ctx interface{}, req interface{}) *MockDeployer_SubmitEvaluation_Call { + return &MockDeployer_SubmitEvaluation_Call{Call: _e.mock.On("SubmitEvaluation", ctx, req)} +} + +func (_c *MockDeployer_SubmitEvaluation_Call) Run(run func(ctx context.Context, req types.EvaluationReq)) *MockDeployer_SubmitEvaluation_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(types.EvaluationReq)) + }) + return _c +} + +func (_c *MockDeployer_SubmitEvaluation_Call) Return(_a0 *types.ArgoWorkFlowRes, _a1 error) *MockDeployer_SubmitEvaluation_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockDeployer_SubmitEvaluation_Call) RunAndReturn(run func(context.Context, types.EvaluationReq) (*types.ArgoWorkFlowRes, error)) *MockDeployer_SubmitEvaluation_Call { + _c.Call.Return(run) + return _c +} + // UpdateCluster provides a mock function with given fields: ctx, data func (_m *MockDeployer) UpdateCluster(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error) { ret := _m.Called(ctx, data) diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go new file mode 100644 index 00000000..a392d894 --- /dev/null +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_ArgoWorkFlowStore.go @@ -0,0 +1,384 @@ +// Code generated by mockery v2.49.1. DO NOT EDIT. + +package database + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + database "opencsg.com/csghub-server/builder/store/database" +) + +// MockArgoWorkFlowStore is an autogenerated mock type for the ArgoWorkFlowStore type +type MockArgoWorkFlowStore struct { + mock.Mock +} + +type MockArgoWorkFlowStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockArgoWorkFlowStore) EXPECT() *MockArgoWorkFlowStore_Expecter { + return &MockArgoWorkFlowStore_Expecter{mock: &_m.Mock} +} + +// CreateWorkFlow provides a mock function with given fields: ctx, workFlow +func (_m *MockArgoWorkFlowStore) CreateWorkFlow(ctx context.Context, workFlow database.ArgoWorkflow) (*database.ArgoWorkflow, error) { + ret := _m.Called(ctx, workFlow) + + if len(ret) == 0 { + panic("no return value specified for CreateWorkFlow") + } + + var r0 *database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)); ok { + return rf(ctx, workFlow) + } + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) *database.ArgoWorkflow); ok { + r0 = rf(ctx, workFlow) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.ArgoWorkflow) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, database.ArgoWorkflow) error); ok { + r1 = rf(ctx, workFlow) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_CreateWorkFlow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateWorkFlow' +type MockArgoWorkFlowStore_CreateWorkFlow_Call struct { + *mock.Call +} + +// CreateWorkFlow is a helper method to define mock.On call +// - ctx context.Context +// - workFlow database.ArgoWorkflow +func (_e *MockArgoWorkFlowStore_Expecter) CreateWorkFlow(ctx interface{}, workFlow interface{}) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + return &MockArgoWorkFlowStore_CreateWorkFlow_Call{Call: _e.mock.On("CreateWorkFlow", ctx, workFlow)} +} + +func (_c *MockArgoWorkFlowStore_CreateWorkFlow_Call) Run(run func(ctx context.Context, workFlow database.ArgoWorkflow)) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(database.ArgoWorkflow)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_CreateWorkFlow_Call) Return(_a0 *database.ArgoWorkflow, _a1 error) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockArgoWorkFlowStore_CreateWorkFlow_Call) RunAndReturn(run func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_CreateWorkFlow_Call { + _c.Call.Return(run) + return _c +} + +// DeleteWorkFlow provides a mock function with given fields: ctx, id +func (_m *MockArgoWorkFlowStore) DeleteWorkFlow(ctx context.Context, id int64) error { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for DeleteWorkFlow") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int64) error); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// MockArgoWorkFlowStore_DeleteWorkFlow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteWorkFlow' +type MockArgoWorkFlowStore_DeleteWorkFlow_Call struct { + *mock.Call +} + +// DeleteWorkFlow is a helper method to define mock.On call +// - ctx context.Context +// - id int64 +func (_e *MockArgoWorkFlowStore_Expecter) DeleteWorkFlow(ctx interface{}, id interface{}) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + return &MockArgoWorkFlowStore_DeleteWorkFlow_Call{Call: _e.mock.On("DeleteWorkFlow", ctx, id)} +} + +func (_c *MockArgoWorkFlowStore_DeleteWorkFlow_Call) Run(run func(ctx context.Context, id int64)) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_DeleteWorkFlow_Call) Return(_a0 error) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockArgoWorkFlowStore_DeleteWorkFlow_Call) RunAndReturn(run func(context.Context, int64) error) *MockArgoWorkFlowStore_DeleteWorkFlow_Call { + _c.Call.Return(run) + return _c +} + +// FindByID provides a mock function with given fields: ctx, id +func (_m *MockArgoWorkFlowStore) FindByID(ctx context.Context, id int64) (database.ArgoWorkflow, error) { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for FindByID") + } + + var r0 database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, int64) (database.ArgoWorkflow, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, int64) database.ArgoWorkflow); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Get(0).(database.ArgoWorkflow) + } + + if rf, ok := ret.Get(1).(func(context.Context, int64) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_FindByID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByID' +type MockArgoWorkFlowStore_FindByID_Call struct { + *mock.Call +} + +// FindByID is a helper method to define mock.On call +// - ctx context.Context +// - id int64 +func (_e *MockArgoWorkFlowStore_Expecter) FindByID(ctx interface{}, id interface{}) *MockArgoWorkFlowStore_FindByID_Call { + return &MockArgoWorkFlowStore_FindByID_Call{Call: _e.mock.On("FindByID", ctx, id)} +} + +func (_c *MockArgoWorkFlowStore_FindByID_Call) Run(run func(ctx context.Context, id int64)) *MockArgoWorkFlowStore_FindByID_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(int64)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByID_Call) Return(WorkFlow database.ArgoWorkflow, err error) *MockArgoWorkFlowStore_FindByID_Call { + _c.Call.Return(WorkFlow, err) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByID_Call) RunAndReturn(run func(context.Context, int64) (database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_FindByID_Call { + _c.Call.Return(run) + return _c +} + +// FindByTaskID provides a mock function with given fields: ctx, id +func (_m *MockArgoWorkFlowStore) FindByTaskID(ctx context.Context, id string) (database.ArgoWorkflow, error) { + ret := _m.Called(ctx, id) + + if len(ret) == 0 { + panic("no return value specified for FindByTaskID") + } + + var r0 database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string) (database.ArgoWorkflow, error)); ok { + return rf(ctx, id) + } + if rf, ok := ret.Get(0).(func(context.Context, string) database.ArgoWorkflow); ok { + r0 = rf(ctx, id) + } else { + r0 = ret.Get(0).(database.ArgoWorkflow) + } + + if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { + r1 = rf(ctx, id) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_FindByTaskID_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByTaskID' +type MockArgoWorkFlowStore_FindByTaskID_Call struct { + *mock.Call +} + +// FindByTaskID is a helper method to define mock.On call +// - ctx context.Context +// - id string +func (_e *MockArgoWorkFlowStore_Expecter) FindByTaskID(ctx interface{}, id interface{}) *MockArgoWorkFlowStore_FindByTaskID_Call { + return &MockArgoWorkFlowStore_FindByTaskID_Call{Call: _e.mock.On("FindByTaskID", ctx, id)} +} + +func (_c *MockArgoWorkFlowStore_FindByTaskID_Call) Run(run func(ctx context.Context, id string)) *MockArgoWorkFlowStore_FindByTaskID_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByTaskID_Call) Return(WorkFlow database.ArgoWorkflow, err error) *MockArgoWorkFlowStore_FindByTaskID_Call { + _c.Call.Return(WorkFlow, err) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByTaskID_Call) RunAndReturn(run func(context.Context, string) (database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_FindByTaskID_Call { + _c.Call.Return(run) + return _c +} + +// FindByUsername provides a mock function with given fields: ctx, username, per, page +func (_m *MockArgoWorkFlowStore) FindByUsername(ctx context.Context, username string, per int, page int) ([]database.ArgoWorkflow, int, error) { + ret := _m.Called(ctx, username, per, page) + + if len(ret) == 0 { + panic("no return value specified for FindByUsername") + } + + var r0 []database.ArgoWorkflow + var r1 int + var r2 error + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) ([]database.ArgoWorkflow, int, error)); ok { + return rf(ctx, username, per, page) + } + if rf, ok := ret.Get(0).(func(context.Context, string, int, int) []database.ArgoWorkflow); ok { + r0 = rf(ctx, username, per, page) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]database.ArgoWorkflow) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, int, int) int); ok { + r1 = rf(ctx, username, per, page) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(context.Context, string, int, int) error); ok { + r2 = rf(ctx, username, per, page) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 +} + +// MockArgoWorkFlowStore_FindByUsername_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByUsername' +type MockArgoWorkFlowStore_FindByUsername_Call struct { + *mock.Call +} + +// FindByUsername is a helper method to define mock.On call +// - ctx context.Context +// - username string +// - per int +// - page int +func (_e *MockArgoWorkFlowStore_Expecter) FindByUsername(ctx interface{}, username interface{}, per interface{}, page interface{}) *MockArgoWorkFlowStore_FindByUsername_Call { + return &MockArgoWorkFlowStore_FindByUsername_Call{Call: _e.mock.On("FindByUsername", ctx, username, per, page)} +} + +func (_c *MockArgoWorkFlowStore_FindByUsername_Call) Run(run func(ctx context.Context, username string, per int, page int)) *MockArgoWorkFlowStore_FindByUsername_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(int), args[3].(int)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByUsername_Call) Return(WorkFlows []database.ArgoWorkflow, total int, err error) *MockArgoWorkFlowStore_FindByUsername_Call { + _c.Call.Return(WorkFlows, total, err) + return _c +} + +func (_c *MockArgoWorkFlowStore_FindByUsername_Call) RunAndReturn(run func(context.Context, string, int, int) ([]database.ArgoWorkflow, int, error)) *MockArgoWorkFlowStore_FindByUsername_Call { + _c.Call.Return(run) + return _c +} + +// UpdateWorkFlow provides a mock function with given fields: ctx, workFlow +func (_m *MockArgoWorkFlowStore) UpdateWorkFlow(ctx context.Context, workFlow database.ArgoWorkflow) (*database.ArgoWorkflow, error) { + ret := _m.Called(ctx, workFlow) + + if len(ret) == 0 { + panic("no return value specified for UpdateWorkFlow") + } + + var r0 *database.ArgoWorkflow + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)); ok { + return rf(ctx, workFlow) + } + if rf, ok := ret.Get(0).(func(context.Context, database.ArgoWorkflow) *database.ArgoWorkflow); ok { + r0 = rf(ctx, workFlow) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.ArgoWorkflow) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, database.ArgoWorkflow) error); ok { + r1 = rf(ctx, workFlow) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockArgoWorkFlowStore_UpdateWorkFlow_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpdateWorkFlow' +type MockArgoWorkFlowStore_UpdateWorkFlow_Call struct { + *mock.Call +} + +// UpdateWorkFlow is a helper method to define mock.On call +// - ctx context.Context +// - workFlow database.ArgoWorkflow +func (_e *MockArgoWorkFlowStore_Expecter) UpdateWorkFlow(ctx interface{}, workFlow interface{}) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + return &MockArgoWorkFlowStore_UpdateWorkFlow_Call{Call: _e.mock.On("UpdateWorkFlow", ctx, workFlow)} +} + +func (_c *MockArgoWorkFlowStore_UpdateWorkFlow_Call) Run(run func(ctx context.Context, workFlow database.ArgoWorkflow)) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(database.ArgoWorkflow)) + }) + return _c +} + +func (_c *MockArgoWorkFlowStore_UpdateWorkFlow_Call) Return(_a0 *database.ArgoWorkflow, _a1 error) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockArgoWorkFlowStore_UpdateWorkFlow_Call) RunAndReturn(run func(context.Context, database.ArgoWorkflow) (*database.ArgoWorkflow, error)) *MockArgoWorkFlowStore_UpdateWorkFlow_Call { + _c.Call.Return(run) + return _c +} + +// NewMockArgoWorkFlowStore creates a new instance of MockArgoWorkFlowStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockArgoWorkFlowStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockArgoWorkFlowStore { + mock := &MockArgoWorkFlowStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go new file mode 100644 index 00000000..a149c0c0 --- /dev/null +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagRuleStore.go @@ -0,0 +1,99 @@ +// Code generated by mockery v2.49.1. DO NOT EDIT. + +package database + +import ( + context "context" + + mock "github.com/stretchr/testify/mock" + database "opencsg.com/csghub-server/builder/store/database" +) + +// MockTagRuleStore is an autogenerated mock type for the TagRuleStore type +type MockTagRuleStore struct { + mock.Mock +} + +type MockTagRuleStore_Expecter struct { + mock *mock.Mock +} + +func (_m *MockTagRuleStore) EXPECT() *MockTagRuleStore_Expecter { + return &MockTagRuleStore_Expecter{mock: &_m.Mock} +} + +// FindByRepo provides a mock function with given fields: ctx, category, namespace, repoName, repoType +func (_m *MockTagRuleStore) FindByRepo(ctx context.Context, category string, namespace string, repoName string, repoType string) (*database.TagRule, error) { + ret := _m.Called(ctx, category, namespace, repoName, repoType) + + if len(ret) == 0 { + panic("no return value specified for FindByRepo") + } + + var r0 *database.TagRule + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) (*database.TagRule, error)); ok { + return rf(ctx, category, namespace, repoName, repoType) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string, string) *database.TagRule); ok { + r0 = rf(ctx, category, namespace, repoName, repoType) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.TagRule) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string, string) error); ok { + r1 = rf(ctx, category, namespace, repoName, repoType) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTagRuleStore_FindByRepo_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindByRepo' +type MockTagRuleStore_FindByRepo_Call struct { + *mock.Call +} + +// FindByRepo is a helper method to define mock.On call +// - ctx context.Context +// - category string +// - namespace string +// - repoName string +// - repoType string +func (_e *MockTagRuleStore_Expecter) FindByRepo(ctx interface{}, category interface{}, namespace interface{}, repoName interface{}, repoType interface{}) *MockTagRuleStore_FindByRepo_Call { + return &MockTagRuleStore_FindByRepo_Call{Call: _e.mock.On("FindByRepo", ctx, category, namespace, repoName, repoType)} +} + +func (_c *MockTagRuleStore_FindByRepo_Call) Run(run func(ctx context.Context, category string, namespace string, repoName string, repoType string)) *MockTagRuleStore_FindByRepo_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string), args[4].(string)) + }) + return _c +} + +func (_c *MockTagRuleStore_FindByRepo_Call) Return(_a0 *database.TagRule, _a1 error) *MockTagRuleStore_FindByRepo_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTagRuleStore_FindByRepo_Call) RunAndReturn(run func(context.Context, string, string, string, string) (*database.TagRule, error)) *MockTagRuleStore_FindByRepo_Call { + _c.Call.Return(run) + return _c +} + +// NewMockTagRuleStore creates a new instance of MockTagRuleStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockTagRuleStore(t interface { + mock.TestingT + Cleanup(func()) +}) *MockTagRuleStore { + mock := &MockTagRuleStore{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go index 862b878d..f2638d83 100644 --- a/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go +++ b/_mocks/opencsg.com/csghub-server/builder/store/database/mock_TagStore.go @@ -900,6 +900,67 @@ func (_c *MockTagStore_FindOrCreate_Call) RunAndReturn(run func(context.Context, return _c } +// FindTag provides a mock function with given fields: ctx, name, scope, category +func (_m *MockTagStore) FindTag(ctx context.Context, name string, scope string, category string) (*database.Tag, error) { + ret := _m.Called(ctx, name, scope, category) + + if len(ret) == 0 { + panic("no return value specified for FindTag") + } + + var r0 *database.Tag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) (*database.Tag, error)); ok { + return rf(ctx, name, scope, category) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string, string) *database.Tag); ok { + r0 = rf(ctx, name, scope, category) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*database.Tag) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string, string) error); ok { + r1 = rf(ctx, name, scope, category) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTagStore_FindTag_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindTag' +type MockTagStore_FindTag_Call struct { + *mock.Call +} + +// FindTag is a helper method to define mock.On call +// - ctx context.Context +// - name string +// - scope string +// - category string +func (_e *MockTagStore_Expecter) FindTag(ctx interface{}, name interface{}, scope interface{}, category interface{}) *MockTagStore_FindTag_Call { + return &MockTagStore_FindTag_Call{Call: _e.mock.On("FindTag", ctx, name, scope, category)} +} + +func (_c *MockTagStore_FindTag_Call) Run(run func(ctx context.Context, name string, scope string, category string)) *MockTagStore_FindTag_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(string)) + }) + return _c +} + +func (_c *MockTagStore_FindTag_Call) Return(_a0 *database.Tag, _a1 error) *MockTagStore_FindTag_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTagStore_FindTag_Call) RunAndReturn(run func(context.Context, string, string, string) (*database.Tag, error)) *MockTagStore_FindTag_Call { + _c.Call.Return(run) + return _c +} + // GetTagsByScopeAndCategories provides a mock function with given fields: ctx, scope, categories func (_m *MockTagStore) GetTagsByScopeAndCategories(ctx context.Context, scope database.TagScope, categories []string) ([]*database.Tag, error) { ret := _m.Called(ctx, scope, categories) diff --git a/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go b/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go index 76b73284..af076cd7 100644 --- a/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go +++ b/_mocks/opencsg.com/csghub-server/component/mock_TagComponent.go @@ -82,6 +82,66 @@ func (_c *MockTagComponent_AllTags_Call) RunAndReturn(run func(context.Context) return _c } +// AllTagsByScopeAndCategory provides a mock function with given fields: ctx, scope, category +func (_m *MockTagComponent) AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) { + ret := _m.Called(ctx, scope, category) + + if len(ret) == 0 { + panic("no return value specified for AllTagsByScopeAndCategory") + } + + var r0 []*database.Tag + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, string, string) ([]*database.Tag, error)); ok { + return rf(ctx, scope, category) + } + if rf, ok := ret.Get(0).(func(context.Context, string, string) []*database.Tag); ok { + r0 = rf(ctx, scope, category) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*database.Tag) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, scope, category) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockTagComponent_AllTagsByScopeAndCategory_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AllTagsByScopeAndCategory' +type MockTagComponent_AllTagsByScopeAndCategory_Call struct { + *mock.Call +} + +// AllTagsByScopeAndCategory is a helper method to define mock.On call +// - ctx context.Context +// - scope string +// - category string +func (_e *MockTagComponent_Expecter) AllTagsByScopeAndCategory(ctx interface{}, scope interface{}, category interface{}) *MockTagComponent_AllTagsByScopeAndCategory_Call { + return &MockTagComponent_AllTagsByScopeAndCategory_Call{Call: _e.mock.On("AllTagsByScopeAndCategory", ctx, scope, category)} +} + +func (_c *MockTagComponent_AllTagsByScopeAndCategory_Call) Run(run func(ctx context.Context, scope string, category string)) *MockTagComponent_AllTagsByScopeAndCategory_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *MockTagComponent_AllTagsByScopeAndCategory_Call) Return(_a0 []*database.Tag, _a1 error) *MockTagComponent_AllTagsByScopeAndCategory_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockTagComponent_AllTagsByScopeAndCategory_Call) RunAndReturn(run func(context.Context, string, string) ([]*database.Tag, error)) *MockTagComponent_AllTagsByScopeAndCategory_Call { + _c.Call.Return(run) + return _c +} + // ClearMetaTags provides a mock function with given fields: ctx, repoType, namespace, name func (_m *MockTagComponent) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace string, name string) error { ret := _m.Called(ctx, repoType, namespace, name) diff --git a/api/handler/evaluation.go b/api/handler/evaluation.go new file mode 100644 index 00000000..74522220 --- /dev/null +++ b/api/handler/evaluation.go @@ -0,0 +1,147 @@ +package handler + +import ( + "errors" + "fmt" + "log/slog" + "strconv" + + "github.com/gin-gonic/gin" + "opencsg.com/csghub-server/api/httpbase" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/component" +) + +func NewEvaluationHandler(config *config.Config) (*EvaluationHandler, error) { + wkf, err := component.NewEvaluationComponent(config) + if err != nil { + return nil, err + } + sc, err := component.NewSensitiveComponent(config) + if err != nil { + return nil, fmt.Errorf("error creating sensitive component:%w", err) + } + return &EvaluationHandler{ + c: wkf, + sc: sc, + }, nil +} + +type EvaluationHandler struct { + c component.EvaluationComponent + sc component.SensitiveComponent +} + +// create evaluation godoc +// @Security ApiKey +// @Summary run model evaluation +// @Tags Evaluation +// @Accept json +// @Produce json +// @Param namespace path string true "namespace" +// @Param name path string true "name" +// @Param body body types.EvaluationReq true "body setting of evaluation" +// @Success 200 {object} string "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /evaluations [post] +func (h *EvaluationHandler) RunEvaluation(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + + var req types.EvaluationReq + if err := ctx.ShouldBindJSON(&req); err != nil { + slog.Error("Bad request format", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + _, err := h.sc.CheckRequestV2(ctx, &req) + if err != nil { + slog.Error("failed to check sensitive request", slog.Any("error", err)) + httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error()) + return + } + req.Username = currentUser + evaluation, err := h.c.CreateEvaluation(ctx, req) + if err != nil { + slog.Error("Failed to create evaluation job", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + httpbase.OK(ctx, evaluation) +} + +// get evaluation godoc +// @Security ApiKey +// @Summary get model evaluation +// @Tags Evaluation +// @Accept json +// @Produce json +// @Param id path string true "id" +// @Success 200 {object} types.EvaluationRes "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /evaluations/{id} [get] +func (h *EvaluationHandler) GetEvaluation(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + id, err := strconv.ParseInt(ctx.Param("id"), 10, 64) + if err != nil { + slog.Error("Bad request format", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + var req = &types.EvaluationGetReq{} + req.ID = id + req.Username = currentUser + evaluation, err := h.c.GetEvaluation(ctx, *req) + if err != nil { + slog.Error("Failed to get evaluation job", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + httpbase.OK(ctx, evaluation) + +} + +// deleteEvaluation godoc +// @Security ApiKey +// @Summary delete model evaluation +// @Tags Evaluation +// @Accept json +// @Produce json +// @Param id path string true "id" +// @Success 200 {object} string "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /evaluations/{id} [delete] +func (h *EvaluationHandler) DeleteEvaluation(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + id, err := strconv.ParseInt(ctx.Param("id"), 10, 64) + if err != nil { + slog.Error("Bad request format", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + var req = &types.EvaluationDelReq{} + req.ID = id + req.Username = currentUser + err = h.c.DeleteEvaluation(ctx, *req) + if err != nil { + slog.Error("Failed to delete evaluation job", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + httpbase.OK(ctx, nil) +} diff --git a/api/handler/repo.go b/api/handler/repo.go index 04b77443..07ff9bfb 100644 --- a/api/handler/repo.go +++ b/api/handler/repo.go @@ -1209,7 +1209,7 @@ func (h *RepoHandler) DeleteMirror(ctx *gin.Context) { // @Param namespace path string true "namespace" // @Param name path string true "name" // @Param current_user query string false "current user" -// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2) default(1) +// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2, 4) default(1) // @Success 200 {object} string "OK" // @Failure 400 {object} types.APIBadRequest "Bad request" // @Failure 500 {object} types.APIInternalServerError "Internal server error" diff --git a/api/handler/tag.go b/api/handler/tag.go index a707bd06..e4e62ca2 100644 --- a/api/handler/tag.go +++ b/api/handler/tag.go @@ -31,13 +31,16 @@ type TagsHandler struct { // @Tags Tag // @Accept json // @Produce json +// @Param category query string false "category name" +// @Param scope query string false "scope name" Enums(model, dataset) // @Success 200 {object} types.ResponseWithTotal{data=[]database.Tag,total=int} "tags" // @Failure 500 {object} types.APIInternalServerError "Internal server error" // @Router /tags [get] func (t *TagsHandler) AllTags(ctx *gin.Context) { //TODO:validate inputs - - tags, err := t.tc.AllTags(ctx) + category := ctx.Query("category") + scope := ctx.Query("scope") + tags, err := t.tc.AllTagsByScopeAndCategory(ctx, scope, category) if err != nil { slog.Error("Failed to load tags", "error", err) httpbase.ServerError(ctx, err) diff --git a/api/handler/user.go b/api/handler/user.go index b0454146..c062710f 100644 --- a/api/handler/user.go +++ b/api/handler/user.go @@ -854,3 +854,49 @@ func (h *UserHandler) Prompts(ctx *gin.Context) { } ctx.JSON(http.StatusOK, respData) } + +// GetUserEvaluations godoc +// @Security ApiKey +// @Summary Get user evaluations +// @Description get user evaluations +// @Tags User +// @Accept json +// @Produce json +// @Param username path string true "username" +// @Param per query int false "per" default(20) +// @Param page query int false "per page" default(1) +// @Success 200 {object} types.Response{} "OK" +// @Failure 400 {object} types.APIBadRequest "Bad request" +// @Failure 500 {object} types.APIInternalServerError "Internal server error" +// @Router /user/{username}/evaluations [get] +func (h *UserHandler) GetEvaluations(ctx *gin.Context) { + currentUser := httpbase.GetCurrentUser(ctx) + if currentUser == "" { + httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first")) + return + } + var req types.UserEvaluationReq + per, page, err := common.GetPerAndPageFromContext(ctx) + if err != nil { + slog.Error("Bad request format of page and per", "error", err) + httpbase.BadRequest(ctx, err.Error()) + return + } + + req.Owner = ctx.Param("username") + req.CurrentUser = httpbase.GetCurrentUser(ctx) + req.Page = page + req.PageSize = per + ds, total, err := h.c.Evaluations(ctx, &req) + if err != nil { + slog.Error("Failed to get user evaluations", slog.Any("error", err)) + httpbase.ServerError(ctx, err) + return + } + + respData := gin.H{ + "data": ds, + "total": total, + } + ctx.JSON(http.StatusOK, respData) +} diff --git a/api/router/api.go b/api/router/api.go index 670b6f19..9b01c3e8 100644 --- a/api/router/api.go +++ b/api/router/api.go @@ -120,8 +120,17 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { apiGroup.POST("/list/spaces_by_path", cache.CacheByRequestURI(memoryStore, 1*time.Minute), listHandler.ListSpacesByPath) } + //evaluation handler + evaluationHandler, err := handler.NewEvaluationHandler(config) + if err != nil { + return nil, fmt.Errorf("error creatring evaluation handler: %v", err) + } + + createEvaluationRoutes(apiGroup, evaluationHandler) + // Model routes createModelRoutes(config, apiGroup, needAPIKey, modelHandler, repoCommonHandler) + // Dataset routes createDatasetRoutes(config, apiGroup, dsHandler, repoCommonHandler) @@ -132,13 +141,6 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { // Code routes createCodeRoutes(config, apiGroup, codeHandler, repoCommonHandler) - // Dataset viewer - dsViewerHandler, err := handler.NewDatasetViewerHandler(config) - if err != nil { - return nil, fmt.Errorf("error creating dataset viewer handler:%w", err) - } - apiGroup.GET("/datasets/:namespace/:name/viewer/*file_path", dsViewerHandler.View) - spaceHandler, err := handler.NewSpaceHandler(config) if err != nil { return nil, fmt.Errorf("error creating space handler:%w", err) @@ -386,6 +388,16 @@ func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) { return r, nil } +func createEvaluationRoutes(apiGroup *gin.RouterGroup, evaluationHandler *handler.EvaluationHandler) { + // Models routes + evaluationsGroup := apiGroup.Group("/evaluations") + { + evaluationsGroup.POST("", evaluationHandler.RunEvaluation) + evaluationsGroup.DELETE("/:id", evaluationHandler.DeleteEvaluation) + evaluationsGroup.GET("/:id", evaluationHandler.GetEvaluation) + } +} + func createModelRoutes(config *config.Config, apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, modelHandler *handler.ModelHandler, repoCommonHandler *handler.RepoHandler) { // Models routes modelsGroup := apiGroup.Group("/models") @@ -648,6 +660,8 @@ func createUserRoutes(apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, use apiGroup.GET("/user/:username/likes/datasets", userHandler.LikesDatasets) apiGroup.GET("/user/:username/run/:repo_type", userHandler.GetRunDeploys) apiGroup.GET("/user/:username/finetune/instances", userHandler.GetFinetuneInstances) + // User evaluations + apiGroup.GET("/user/:username/evaluations", userHandler.GetEvaluations) } // User collection diff --git a/builder/deploy/cluster/cluster_manager.go b/builder/deploy/cluster/cluster_manager.go index a5692c5b..b87e25be 100644 --- a/builder/deploy/cluster/cluster_manager.go +++ b/builder/deploy/cluster/cluster_manager.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" + "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,6 +28,7 @@ type Cluster struct { ConfigPath string // Path to the kubeconfig file Client *kubernetes.Clientset // Kubernetes client KnativeClient *knative.Clientset // Knative client + ArgoClient *versioned.Clientset // Argo client StorageClass string } @@ -61,6 +63,10 @@ func NewClusterPool() (*ClusterPool, error) { if err != nil { return nil, err } + argoClient, err := versioned.NewForConfig(config) + if err != nil { + return nil, err + } knativeClient, err := knative.NewForConfig(config) if err != nil { slog.Error("falied to create knative client", "error", err) @@ -72,6 +78,7 @@ func NewClusterPool() (*ClusterPool, error) { ConfigPath: kubeconfig, Client: client, KnativeClient: knativeClient, + ArgoClient: argoClient, }) err = pool.ClusterStore.Add(context.TODO(), id, fmt.Sprintf("region-%d", i)) if err != nil { diff --git a/builder/deploy/common/deploy_config.go b/builder/deploy/common/deploy_config.go index 53d67417..4ea02f81 100644 --- a/builder/deploy/common/deploy_config.go +++ b/builder/deploy/common/deploy_config.go @@ -13,5 +13,6 @@ type DeployConfig struct { PublicRootDomain string SSHDomain string //download lfs object from internal s3 address - S3Internal bool + S3Internal bool + IsMasterHost bool } diff --git a/builder/deploy/deployer.go b/builder/deploy/deployer.go index d20c17d5..8e05d81a 100644 --- a/builder/deploy/deployer.go +++ b/builder/deploy/deployer.go @@ -39,6 +39,10 @@ type Deployer interface { UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq, deploy *database.Deploy) error StartDeploy(ctx context.Context, deploy *database.Deploy) error CheckResourceAvailable(ctx context.Context, clusterId string, hardWare *types.HardWare) (bool, error) + SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) + ListEvaluations(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) + DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error + GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) } var _ Deployer = (*deployer)(nil) @@ -86,7 +90,6 @@ func newDeployer(s scheduler.Scheduler, ib imagebuilder.Builder, ir imagerunner. } }() go d.startAccounting() - return d, nil } @@ -775,3 +778,67 @@ func CheckResource(clusterResources *types.ClusterRes, hardware *types.HardWare) } return false } + +// SubmitEvaluation +func (d *deployer) SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + env := make(map[string]string) + env["REVISION"] = "main" + env["MODEL_ID"] = req.ModelId + env["DATASET_IDS"] = strings.Join(req.Datasets, ",") + env["REVISION"] = "main" + env["ACCESS_TOKEN"] = req.Token + env["HF_ENDPOINT"] = req.DownloadEndpoint + + if req.Hardware.Gpu.Num != "" { + env["GPU_NUM"] = req.Hardware.Gpu.Num + } + templates := []types.ArgoFlowTemplate{} + templates = append(templates, types.ArgoFlowTemplate{ + Name: "evaluation", + Env: env, + HardWare: req.Hardware, + Image: req.Image, + }, + ) + uniqueFlowName := d.sfNode.Generate().Base36() + flowReq := &types.ArgoWorkFlowReq{ + TaskName: req.TaskName, + TaskId: uniqueFlowName, + TaskType: req.TaskType, + TaskDesc: req.TaskDesc, + Image: req.Image, + Datasets: req.Datasets, + Username: req.Username, + UserUUID: req.UserUUID, + RepoIds: []string{req.ModelId}, + Entrypoint: "evaluation", + ClusterID: req.ClusterID, + Templates: templates, + RepoType: req.RepoType, + ResourceId: req.ResourceId, + ResourceName: req.ResourceName, + } + if req.ResourceId == 0 { + flowReq.ShareMode = true + } + return d.ir.SubmitWorkFlow(ctx, flowReq) +} +func (d *deployer) ListEvaluations(ctx context.Context, username string, per int, page int) (*types.ArgoWorkFlowListRes, error) { + return d.ir.ListWorkFlows(ctx, username, per, page) +} + +func (d *deployer) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { + _, err := d.ir.DeleteWorkFlow(ctx, req) + if err != nil { + return err + } + return nil +} + +func (d *deployer) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) { + wf, err := d.ir.GetWorkFlow(ctx, req) + if err != nil { + return nil, err + } + return wf, err +} diff --git a/builder/deploy/imagerunner/local_runner.go b/builder/deploy/imagerunner/local_runner.go index 4afa391c..9c0fd904 100644 --- a/builder/deploy/imagerunner/local_runner.go +++ b/builder/deploy/imagerunner/local_runner.go @@ -3,6 +3,7 @@ package imagerunner import ( "context" + "opencsg.com/csghub-server/api/httpbase" "opencsg.com/csghub-server/builder/deploy/common" "opencsg.com/csghub-server/common/types" ) @@ -89,3 +90,19 @@ func (h *LocalRunner) GetClusterById(ctx context.Context, clusterId string) (*ty func (h *LocalRunner) UpdateCluster(ctx context.Context, data *types.ClusterRequest) (*types.UpdateClusterResponse, error) { return nil, nil } + +func (h *LocalRunner) SubmitWorkFlow(ctx context.Context, req *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) { + return nil, nil +} + +func (h *LocalRunner) ListWorkFlows(ctx context.Context, username string, per, page int) (*types.ArgoWorkFlowListRes, error) { + return nil, nil +} + +func (h *LocalRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) { + return nil, nil +} + +func (h *LocalRunner) GetWorkFlow(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) { + return nil, nil +} diff --git a/builder/deploy/imagerunner/remote_runner.go b/builder/deploy/imagerunner/remote_runner.go index 7b02a4e6..fca1e22b 100644 --- a/builder/deploy/imagerunner/remote_runner.go +++ b/builder/deploy/imagerunner/remote_runner.go @@ -12,6 +12,7 @@ import ( "net/url" "time" + "opencsg.com/csghub-server/api/httpbase" "opencsg.com/csghub-server/common/types" ) @@ -315,3 +316,68 @@ func (h *RemoteRunner) UpdateCluster(ctx context.Context, data *types.ClusterReq return &resp, nil } + +// submit argo workflow +func (h *RemoteRunner) SubmitWorkFlow(ctx context.Context, req *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) { + url := fmt.Sprintf("%s/api/v1/workflows", h.remote) + // Create a new HTTP client with a timeout + response, err := h.doRequest(http.MethodPost, url, req) + if err != nil { + return nil, fmt.Errorf("failed to submit evaluation job, %w", err) + } + defer response.Body.Close() + + var res types.ArgoWorkFlowRes + if err := json.NewDecoder(response.Body).Decode(&res); err != nil { + return nil, err + } + return &res, nil +} + +// list workflows +func (h *RemoteRunner) ListWorkFlows(ctx context.Context, username string, per, page int) (*types.ArgoWorkFlowListRes, error) { + url := fmt.Sprintf("%s/api/v1/workflows?username=%s&per=%d&page=%d", h.remote, username, per, page) + // Create a new HTTP client with a timeout + response, err := h.doRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("failed to list evaluation jobs, %w", err) + } + defer response.Body.Close() + var res types.ArgoWorkFlowListRes + if err := json.NewDecoder(response.Body).Decode(&res); err != nil { + return nil, err + } + return &res, nil +} + +// delete workflow +func (h *RemoteRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) { + url := fmt.Sprintf("%s/api/v1/workflows/%d", h.remote, req.ID) + // Create a new HTTP client with a timeout + response, err := h.doRequest(http.MethodDelete, url, req) + if err != nil { + return nil, fmt.Errorf("failed to delete evaluation job, %w", err) + } + defer response.Body.Close() + var res httpbase.R + err = json.NewDecoder(response.Body).Decode(&res) + if err != nil { + return nil, err + } + return &res, nil +} + +func (h *RemoteRunner) GetWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*types.ArgoWorkFlowRes, error) { + url := fmt.Sprintf("%s/api/v1/workflows/%d", h.remote, req.ID) + response, err := h.doRequest(http.MethodGet, url, req) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var res types.ArgoWorkFlowRes + if err := json.NewDecoder(response.Body).Decode(&res); err != nil { + return nil, err + } + return &res, nil +} diff --git a/builder/deploy/imagerunner/runner.go b/builder/deploy/imagerunner/runner.go index a21a41fa..186abf20 100644 --- a/builder/deploy/imagerunner/runner.go +++ b/builder/deploy/imagerunner/runner.go @@ -3,6 +3,7 @@ package imagerunner import ( "context" + "opencsg.com/csghub-server/api/httpbase" "opencsg.com/csghub-server/common/types" ) @@ -19,4 +20,8 @@ type Runner interface { ListCluster(ctx context.Context) ([]types.ClusterResponse, error) GetClusterById(ctx context.Context, clusterId string) (*types.ClusterResponse, error) UpdateCluster(ctx context.Context, data *types.ClusterRequest) (*types.UpdateClusterResponse, error) + SubmitWorkFlow(context.Context, *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) + ListWorkFlows(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error) + DeleteWorkFlow(context.Context, types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) + GetWorkFlow(context.Context, types.ArgoWorkFlowDeleteReq) (*types.ArgoWorkFlowRes, error) } diff --git a/builder/store/database/argo_workflow.go b/builder/store/database/argo_workflow.go new file mode 100644 index 00000000..4378f5b7 --- /dev/null +++ b/builder/store/database/argo_workflow.go @@ -0,0 +1,121 @@ +package database + +import ( + "context" + "fmt" + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "opencsg.com/csghub-server/common/types" +) + +type argoWorkFlowStoreImpl struct { + db *DB +} + +type ArgoWorkFlowStore interface { + FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) + FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) + FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) + CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) + // mainly for update status + UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) + // delete workflow by id + DeleteWorkFlow(ctx context.Context, id int64) error +} + +func NewArgoWorkFlowStore() ArgoWorkFlowStore { + return &argoWorkFlowStoreImpl{ + db: defaultDB, + } +} + +func NewArgoWorkFlowStoreWithDB(db *DB) ArgoWorkFlowStore { + return &argoWorkFlowStoreImpl{ + db: db, + } +} + +type ArgoWorkflow struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + Username string `bun:",notnull" json:"username"` + UserUUID string `bun:",notnull" json:"user_uuid"` + TaskName string `bun:",notnull" json:"task_name"` // user input name + TaskId string `bun:",notnull" json:"task_id"` // generated task id + TaskType types.TaskType `bun:",notnull" json:"task_type"` + ClusterID string `bun:",notnull" json:"cluster_id"` + Namespace string `bun:",notnull" json:"namespace"` + RepoIds []string `bun:",notnull,type:jsonb" json:"repo_ids"` + RepoType string `bun:",notnull" json:"repo_type"` + TaskDesc string `bun:"," json:"task_desc"` + Status v1alpha1.WorkflowPhase `bun:"," json:"status"` + Reason string `bun:"," json:"reason"` // reason for status + Image string `bun:",notnull" json:"image"` // ArgoWorkFlow framework + Datasets []string `bun:",notnull,type:jsonb" json:"datasets"` + ResourceId int64 `bun:",nullzero" json:"resource_id"` + ResourceName string `bun:"," json:"resource_name"` + SubmitTime time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"submit_time"` + StartTime time.Time `bun:",nullzero" json:"start_time"` + EndTime time.Time `bun:",nullzero" json:"end_time"` + ResultURL string `bun:"," json:"result_url"` + DownloadURL string `bun:"," json:"download_url"` + FailuresURL string `bun:"," json:"failures_url"` +} + +func (s *argoWorkFlowStoreImpl) FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) { + err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("id = ?", id).Scan(ctx, &WorkFlow) + if err != nil { + return + } + return +} + +func (s *argoWorkFlowStoreImpl) FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) { + err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("task_id = ?", id).Scan(ctx, &WorkFlow) + if err != nil { + return + } + return +} + +func (s *argoWorkFlowStoreImpl) FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) { + query := s.db.Operator.Core. + NewSelect(). + Model(&WorkFlows). + Where("username = ?", username) + + query = query.Order("submit_time DESC"). + Limit(per). + Offset((page - 1) * per) + + err = query.Scan(ctx) + if err != nil { + return + } + total, err = query.Count(ctx) + if err != nil { + return + } + return +} + +func (s *argoWorkFlowStoreImpl) CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { + res, err := s.db.Core.NewInsert().Model(&workFlow).Exec(ctx, &workFlow) + if err := assertAffectedOneRow(res, err); err != nil { + return nil, fmt.Errorf("failed to save WorkFlow in db, error:%w", err) + } + + return &workFlow, nil +} + +// mainly for update status +func (s *argoWorkFlowStoreImpl) UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) { + _, err := s.db.Core.NewUpdate().Model(&workFlow).WherePK().Exec(ctx) + return &workFlow, err +} + +// delete workflow by id +func (s *argoWorkFlowStoreImpl) DeleteWorkFlow(ctx context.Context, id int64) error { + _, err := s.db.Core.NewDelete().Model(&ArgoWorkflow{}).Where("id = ?", id).Exec(ctx) + return err +} diff --git a/builder/store/database/migrations/20241105124716_create_table_workflows.go b/builder/store/database/migrations/20241105124716_create_table_workflows.go new file mode 100644 index 00000000..8b8b780a --- /dev/null +++ b/builder/store/database/migrations/20241105124716_create_table_workflows.go @@ -0,0 +1,51 @@ +package migrations + +import ( + "context" + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/uptrace/bun" + "opencsg.com/csghub-server/common/types" +) + +type ArgoWorkflow struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + Username string `bun:",notnull" json:"username"` + UserUUID string `bun:",notnull" json:"user_uuid"` + TaskName string `bun:",notnull" json:"task_name"` // user input name + TaskId string `bun:",notnull" json:"task_id"` // generated task id + TaskType types.TaskType `bun:",notnull" json:"task_type"` + RepoIds []string `bun:",notnull,type:jsonb" json:"repo_ids"` + RepoType string `bun:",notnull" json:"repo_type"` + TaskDesc string `bun:"," json:"task_desc"` + Status v1alpha1.WorkflowPhase `bun:"," json:"status"` + Reason string `bun:"," json:"reason"` // reason for status + Image string `bun:",notnull" json:"image"` // ArgoWorkFlow framework + Datasets []string `bun:",notnull,type:jsonb" json:"datasets"` + ResourceId int64 `bun:",nullzero" json:"resource_id"` + SubmitTime time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"submit_time"` + StartTime time.Time `bun:",nullzero" json:"start_time"` + EndTime time.Time `bun:",nullzero" json:"end_time"` + ResultURL string `bun:",nullzero" json:"result_url"` + DownloadURL string `bun:",nullzero" json:"download_url"` + FailuresURL string `bun:",nullzero" json:"failures_url"` +} + +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + err := createTables(ctx, db, ArgoWorkflow{}) + if err != nil { + return err + } + _, err = db.NewCreateIndex(). + Model((*ArgoWorkflow)(nil)). + Index("idx_workflow_user_uuid"). + Column("username", "task_id"). + Exec(ctx) + return err + + }, func(ctx context.Context, db *bun.DB) error { + return dropTables(ctx, db, ArgoWorkflow{}) + }) +} diff --git a/builder/store/database/migrations/20241111095821_create_tag_rules.go b/builder/store/database/migrations/20241111095821_create_tag_rules.go new file mode 100644 index 00000000..fd9aa288 --- /dev/null +++ b/builder/store/database/migrations/20241111095821_create_tag_rules.go @@ -0,0 +1,35 @@ +package migrations + +import ( + "context" + "time" + + "github.com/uptrace/bun" +) + +type TagRule struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + RepoName string `bun:",notnull" json:"repo_name"` + RepoType string `bun:",notnull" json:"repo_type"` + Category string `bun:",notnull" json:"category"` + TagName string `bun:",notnull" json:"tag_name"` + Tag Tag `bun:",rel:has-one,join:tag_name=name"` + CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"` +} + +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + err := createTables(ctx, db, TagRule{}) + if err != nil { + return err + } + _, err = db.NewCreateIndex(). + Model((*TagRule)(nil)). + Index("idx_dataset_tag_name_type"). + Column("repo_name", "repo_type"). + Exec(ctx) + return err + }, func(ctx context.Context, db *bun.DB) error { + return dropTables(ctx, db, TagRule{}) + }) +} diff --git a/builder/store/database/migrations/20241111095847_init_tag_rule.down.sql b/builder/store/database/migrations/20241111095847_init_tag_rule.down.sql new file mode 100644 index 00000000..cced29bf --- /dev/null +++ b/builder/store/database/migrations/20241111095847_init_tag_rule.down.sql @@ -0,0 +1,9 @@ +SET statement_timeout = 0; + +--bun:split + +Delete from public.tags where name in ('Knowledge','Reasoning','Examination','Understanding','Code','Other'); + +--bun:split + +Delete from public.tag_rules; diff --git a/builder/store/database/migrations/20241111095847_init_tag_rule.up.sql b/builder/store/database/migrations/20241111095847_init_tag_rule.up.sql new file mode 100644 index 00000000..09b4f284 --- /dev/null +++ b/builder/store/database/migrations/20241111095847_init_tag_rule.up.sql @@ -0,0 +1,112 @@ +SET statement_timeout = 0; + +--bun:split + +INSERT INTO public.tag_categories ("name", "scope") VALUES( 'evaluation', 'dataset') ON CONFLICT ("name", scope) DO NOTHING; + +--bun:split +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Knowledge', 'evaluation', '', 'dataset', true, '知识', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Reasoning', 'evaluation', '', 'dataset', true, '推理', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Examination', 'evaluation', '', 'dataset', true, '考试', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Understanding', 'evaluation', '', 'dataset', true, '理解', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Code', 'evaluation', '', 'dataset', true, '代码', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Other', 'evaluation', '', 'dataset', true, '其他', '2024-11-11 10:42:12.939', '2024-11-11 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +--bun:split +ALTER TABLE public.tag_rules ADD CONSTRAINT unique_tag_rules UNIQUE (repo_name, category); +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('wic', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('summedits', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('chid', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('afqmc', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('bustm', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cluewsc', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('wsc', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('winogrande', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('flores', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('iwslt2017', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('tydiqa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xcopa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xlsum', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('leval', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('longbench', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('govreports', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('narrativeqa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('qasper', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('civilcomments', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('crowspairs', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cvalues', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('jigsawmultilingual', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('truthfulqa', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('advglue', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ifeval', 'evaluation', 'Other', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('boolq', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('commonsense_qa', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('natural_question', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('trivia_qa', 'evaluation', 'Knowledge', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmnli', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ocnli', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ocnli_fc', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ax-b', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ax-g', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('rte', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('anli', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xstory_cloze', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('copa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('record', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('hellaswag', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('piqa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('siqa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('math', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('gsm8k', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('theoremqa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('strategy_qa', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('scibench', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('bbh', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('musr', 'evaluation', 'Reasoning', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ceval-exam', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('agieval', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mmlu', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('gaokao-bench', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmmlu', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ai2_arc', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xiezhi', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmb', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mmlu-pro', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('chembench', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mmmlu_lite', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('wikibench', 'evaluation', 'Examination', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('cmrc_dev', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('drcd_dev', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('race', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('openbookqa', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('squad2.0', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('lcsts', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('xsum', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('summscreen', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('eprstmt', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('lambada', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('tnews', 'evaluation', 'Understanding', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('humaneval', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('humanevalx', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('mbpp', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('apps', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('ds1000', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('code_generation_lite', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('execution-v2', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules ("repo_name", category, "tag_name", "repo_type") VALUES('test_generation', 'evaluation', 'Code', 'dataset') ON CONFLICT ("repo_name", category) DO NOTHING; \ No newline at end of file diff --git a/builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql new file mode 100644 index 00000000..722ddf23 --- /dev/null +++ b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.down.sql @@ -0,0 +1,10 @@ +SET statement_timeout = 0; + +--bun:split + +ALTER TABLE public.argo_workflows DROP COLUMN IF EXISTS cluster_id; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS runtime_framework; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS resource_name; + +--bun:split +Delete from public.tags where name='opencompass' and category='runtime_framework'; diff --git a/builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql new file mode 100644 index 00000000..15a0e652 --- /dev/null +++ b/builder/store/database/migrations/20241117095440_update_argo_workflow_table.up.sql @@ -0,0 +1,18 @@ +SET statement_timeout = 0; + +--bun:split + +ALTER TABLE public.argo_workflows ADD COLUMN IF NOT EXISTS cluster_id VARCHAR; +ALTER TABLE public.argo_workflows ADD COLUMN IF NOT EXISTS namespace VARCHAR; +ALTER TABLE public.argo_workflows ADD COLUMN IF NOT EXISTS resource_name VARCHAR; + +--bun:split + +--add opencompass as runtime framework for all tag_rules created in 20241111095847_init_tag_rule.up.sql +--then drop the default the value +ALTER TABLE public.tag_rules ADD COLUMN IF NOT EXISTS runtime_framework VARCHAR DEFAULT 'opencompass'; +ALTER TABLE public.tag_rules ALTER COLUMN runtime_framework DROP DEFAULT; + +--bun:split +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('opencompass', 'runtime_framework', 'evaluation', 'model', true, 'OpenCompass') ON CONFLICT ("name", category, scope) DO NOTHING; +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('opencompass', 'runtime_framework', 'evaluation', 'dataset', true, 'OpenCompass') ON CONFLICT ("name", category, scope) DO NOTHING; diff --git a/builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql b/builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql new file mode 100644 index 00000000..c3b2a0dd --- /dev/null +++ b/builder/store/database/migrations/20241126071538_add_lm_harness_data.down.sql @@ -0,0 +1,10 @@ +SET statement_timeout = 0; + +--bun:split +Delete from public.tag_rules where runtime_framework='lm-evaluation-harness'; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS namespace; +ALTER TABLE public.tag_rules DROP COLUMN IF EXISTS source; +ALTER TABLE public.tag_rules DROP CONSTRAINT unique_tag_rules; +ALTER TABLE public.tag_rules ADD CONSTRAINT unique_tag_rules UNIQUE (repo_name, category); +Delete from public.tags where name='Benchmark'; +Delete from public.tags where name='lm-evaluation-harness'; diff --git a/builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql b/builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql new file mode 100644 index 00000000..a574bafc --- /dev/null +++ b/builder/store/database/migrations/20241126071538_add_lm_harness_data.up.sql @@ -0,0 +1,160 @@ +SET statement_timeout = 0; + +--bun:split +--add opencompass as namespace for all tag_rules created in 20241111095847_init_tag_rule.up.sql +--then drop the default the value +ALTER TABLE public.tag_rules ADD COLUMN IF NOT EXISTS namespace VARCHAR DEFAULT 'opencompass'; +ALTER TABLE public.tag_rules ALTER COLUMN namespace DROP DEFAULT; +ALTER TABLE public.tag_rules ADD COLUMN IF NOT EXISTS source VARCHAR DEFAULT 'ms'; +ALTER TABLE public.tag_rules ALTER COLUMN source DROP DEFAULT; +ALTER TABLE public.tag_rules DROP CONSTRAINT unique_tag_rules; +ALTER TABLE public.tag_rules ADD CONSTRAINT unique_tag_rules UNIQUE (namespace,repo_name, category); + +--bun:split +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('lm-evaluation-harness', 'runtime_framework', 'evaluation', 'model', true, 'lm-evaluation-harness') ON CONFLICT ("name", category, scope) DO NOTHING; +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name) VALUES('lm-evaluation-harness', 'runtime_framework', 'evaluation', 'dataset', true, 'lm-evaluation-harness') ON CONFLICT ("name", category, scope) DO NOTHING; +INSERT INTO public.tags ("name", category, "group", "scope", built_in, show_name, created_at, updated_at) VALUES('Benchmark', 'evaluation', '', 'dataset', true, 'Benchmark', '2024-11-26 10:42:12.939', '2024-11-26 10:42:12.939') ON CONFLICT ("name", category, scope) DO NOTHING; + +--bun:split +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HCAI', 'metabench', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'bigbench', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('lighteval', 'MATH-Hard', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ofir408', 'MedConceptsQA', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('pbevan11', 'EQ-Bench', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyAI2_arc', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyGSM8k', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyHellaswag', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyMMLU', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyTruthfulQA', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tinyBenchmarks', 'tinyWinogrande', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('wis-k', 'instruction-following-eval', 'evaluation', 'Benchmark', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('SaylorTwift', 'bbh', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('TAUR-Lab', 'MuSR', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ambean', 'lingOly', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'anli', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('openai', 'gsm8k', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'NeQA', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'hindsight-neglect-10shot', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'quote-repetition', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('inverse-scaling', 'redefine-math', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('mathemakitten', 'winobias_antistereotype_test_v5', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('qintongli', 'GSM-Plus', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'swag', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('cambridgeltl', 'xcopa', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'anli', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Rowan', 'hellaswag', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ybisk', 'piqa', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('ErnestSDavis', 'winograd_wsc', 'evaluation', 'Reasoning', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('AYueksel', 'TurkishMMLU', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Hennara', 'copa_ar', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Hennara', 'pica_ar', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('LumiOpen', 'arc_challenge_mt', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('TIGER-Lab', 'MMLU-Pro', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_arc', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_hellaswag', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_mmlu', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('alexandrainst', 'm_truthfulqa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'ai2_arc', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'paloma', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google', 'IFEval', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-aqua-rat', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-biology', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-chemistry', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-chinese', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-english', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-geography', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-history', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-mathcloze', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-mathqa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-gaokao-physics', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-jec-qa-ca', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-jec-qa-kd', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-logiqa-en', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-logiqa-zh', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-lsat-ar', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-lsat-lr', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-lsat-rc', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-math', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-sat-en', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-sat-en-without-passage', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hails', 'agieval-sat-math', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('mideind', 'icelandic-arc-challenge', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'math_qa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('openlifescienceai', 'medmcqa', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'sciq', 'evaluation', 'Examination', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HAERAE-HUB', 'HAE_RAE_BENCH', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HAERAE-HUB', 'KMMLU', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HAERAE-HUB', 'KMMLU-HARD', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Idavidrein', 'gpqa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Muennighoff', 'babi', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimgsm', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimgsm-translate-test', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimmlu', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrimmlu-translate-test', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrixnli', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('masakhane', 'afrixnli-translate-test', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google-research-datasets', 'nq_open', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'openbookqa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('tau', 'commonsense_qa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('mandarjoshi', 'trivia_qa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('truthfulqa', 'truthful_qa', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Stanford', 'web_questions', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('community-datasets', 'qa4mre', 'evaluation', 'Knowledge', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('GBaker', 'MedQA-USMLE-4-options-hf', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'BertaQA', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'EusExams', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'EusProficiency', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'EusTrivia', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'MGSM-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'XCOPA-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'wnli-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('HiTZ', 'xnli-eu', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'belebele', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'glue', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('hitachi-nlp', 'FLD.v2', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('maximegmd', 'glianorex', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google-research-datasets', 'paws-x', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'xnli', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('google', 'xquad', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('cimec', 'lambada', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CogComp', 'mc_taco', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'social_i_qa', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('LSDSem', 'story_cloze', 'evaluation', 'Understanding', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_go', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_java', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_javascript', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_php', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_python', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('CM', 'codexglue_code2text_ruby', 'evaluation', 'Code', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'real-toxicity-prompts', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'into-the-unknown', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'memo-trap', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'modus-tollens', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'pattern-matching-suppression', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'repetitive-algebra', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Albertmade', 'sig-figs', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('BSC-LT', 'COPA-es', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('BSC-LT', 'openbookqa-es', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('EleutherAI', 'headqa', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('EleutherAI', 'winogenerated', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('Geralt-Targaryen', 'MELA', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('NeelNanda', 'pile-10k', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'blimp', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('cais', 'wmdp', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('skg', 'toxigen-data', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('wmt', 'wmt14', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('wmt', 'wmt16', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('allenai', 'winogrande', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('aps', 'super_glue', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'blimp', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('facebook', 'xnli', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('nyu-mll', 'glue', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; +INSERT INTO public.tag_rules (namespace, "repo_name", category, "tag_name", "repo_type", runtime_framework, source) VALUES ('IWSLT', 'iwslt2017', 'evaluation', 'Other', 'dataset', 'lm-evaluation-harness', 'hf') ON CONFLICT (namespace, "repo_name", category) DO NOTHING; + + + diff --git a/builder/store/database/repository.go b/builder/store/database/repository.go index e929d22c..084b6773 100644 --- a/builder/store/database/repository.go +++ b/builder/store/database/repository.go @@ -430,11 +430,14 @@ func (s *repoStoreImpl) PublicToUser(ctx context.Context, repoType types.Reposit ) } if len(filter.Tags) > 0 { - q.Join("JOIN repository_tags ON repository.id = repository_tags.repository_id"). - Join("JOIN tags ON repository_tags.tag_id = tags.id") - for _, tag := range filter.Tags { - q.Where("tags.category = ? AND tags.name = ?", tag.Category, tag.Name) + for i, tag := range filter.Tags { + var asRepoTag = fmt.Sprintf("%s%d", "rt", i) + var asTag = fmt.Sprintf("%s%d", "ts", i) + q.Join(fmt.Sprintf("JOIN repository_tags AS %s ON repository.id = %s.repository_id", asRepoTag, asRepoTag)). + Join(fmt.Sprintf("JOIN tags AS %s ON %s.tag_id = %s.id", asTag, asRepoTag, asTag)) + q.Where(fmt.Sprintf("%s.category = ? AND %s.name = ?", asTag, asTag), tag.Category, tag.Name) } + } count, err = q.Count(ctx) diff --git a/builder/store/database/repository_test.go b/builder/store/database/repository_test.go index 45d8db5c..5de852ba 100644 --- a/builder/store/database/repository_test.go +++ b/builder/store/database/repository_test.go @@ -340,7 +340,7 @@ func TestRepoStore_PublicToUserSimple(t *testing.T) { // case 2: two tag repos, _, err = rs.PublicToUser(ctx, repo.RepositoryType, []int64{1}, filter, 20, 1) require.Nil(t, err) - require.Nil(t, repos) + require.NotNil(t, repos) } func TestRepoStore_PublicToUser(t *testing.T) { diff --git a/builder/store/database/seeds/tag_categories.yml b/builder/store/database/seeds/tag_categories.yml index 9f45a2d8..126bc8b2 100644 --- a/builder/store/database/seeds/tag_categories.yml +++ b/builder/store/database/seeds/tag_categories.yml @@ -1,13 +1,15 @@ categories: -- name: task - scope: model -- name: license - scope: model -- name: framework - scope: model -- name: task - scope: dataset -- name: license - scope: dataset -- name: size - scope: dataset + - name: task + scope: model + - name: license + scope: model + - name: framework + scope: model + - name: task + scope: dataset + - name: license + scope: dataset + - name: size + scope: dataset + - name: evaluation + scope: dataset diff --git a/builder/store/database/tag.go b/builder/store/database/tag.go index 3c0b03fa..e0ced7b6 100644 --- a/builder/store/database/tag.go +++ b/builder/store/database/tag.go @@ -39,6 +39,7 @@ type TagStore interface { UpsertRepoTags(ctx context.Context, repoID int64, oldTagIDs, newTagIDs []int64) (err error) RemoveRepoTags(ctx context.Context, repoID int64, tagIDs []int64) (err error) FindOrCreate(ctx context.Context, tag Tag) (*Tag, error) + FindTag(ctx context.Context, name, scope, category string) (*Tag, error) } func NewTagStore() TagStore { @@ -105,9 +106,15 @@ func (ts *tagStoreImpl) AllTagsByScope(ctx context.Context, scope TagScope) ([]* func (ts *tagStoreImpl) AllTagsByScopeAndCategory(ctx context.Context, scope TagScope, category string) ([]*Tag, error) { var tags []*Tag - err := ts.db.Operator.Core.NewSelect().Model(&tags). - Where("scope = ? and category = ?", scope, category). - Scan(ctx) + query := ts.db.Operator.Core.NewSelect().Model(&tags) + if scope != "" { + query.Where("scope = ?", scope) + } + if category != "" { + query.Where("category = ?", category) + } + + err := query.Scan(ctx) if err != nil { return nil, fmt.Errorf("failed to select tags by scope,cause: %w", err) } @@ -213,8 +220,13 @@ func (ts *tagStoreImpl) SetMetaTags(ctx context.Context, repoType types.Reposito } var metaTagIds []int64 + exCategories := map[string]bool{ + "framework": true, + "runtime_framework": true, + "evaluation": true, + } for _, tag := range repo.Tags { - if tag.Category != "framework" { + if !exCategories[tag.Category] { metaTagIds = append(metaTagIds, tag.ID) } } @@ -368,3 +380,16 @@ func (ts *tagStoreImpl) FindOrCreate(ctx context.Context, tag Tag) (*Tag, error) } return &tag, err } + +// find tag by name +func (ts *tagStoreImpl) FindTag(ctx context.Context, name, scope, category string) (*Tag, error) { + var tag Tag + err := ts.db.Operator.Core.NewSelect(). + Model(&tag). + Where("name = ? and scope = ? and category = ?", name, scope, category). + Scan(ctx) + if err != nil { + return nil, err + } + return &tag, nil +} diff --git a/builder/store/database/tag_rule.go b/builder/store/database/tag_rule.go new file mode 100644 index 00000000..e4b90597 --- /dev/null +++ b/builder/store/database/tag_rule.go @@ -0,0 +1,52 @@ +package database + +import ( + "context" + "time" +) + +type tagRuleStoreImpl struct { + db *DB +} + +type TagRuleStore interface { + // find dataset tag by name and related tag with tag name + FindByRepo(ctx context.Context, category, namespace, repoName, repoType string) (*TagRule, error) +} + +func NewTagRuleStore() TagRuleStore { + return &tagRuleStoreImpl{db: defaultDB} +} + +func NewTagRuleStoreWithDB(db *DB) TagRuleStore { + return &tagRuleStoreImpl{db: db} +} + +// internal use for the relation between dataset and tag +type TagRule struct { + ID int64 `bun:",pk,autoincrement" json:"id"` + Namespace string `bun:",notnull" json:"namespace"` + RepoName string `bun:",notnull" json:"repo_name"` + RepoType string `bun:",notnull" json:"repo_type"` + Category string `bun:",notnull" json:"category"` + TagName string `bun:",notnull" json:"tag_name"` + RuntimeFramework string `bun:"," json:"runtime_framework"` + Source string `bun:"," json:"source"` + Tag Tag `bun:",rel:has-one,join:tag_name=name"` + CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"` +} + +// find dataset tag by name and related tag with tag name +func (s *tagRuleStoreImpl) FindByRepo(ctx context.Context, category, namespace, repoName, repoType string) (*TagRule, error) { + var TagRule TagRule + err := s.db.Core.NewSelect(). + Model(&TagRule). + Relation("Tag"). + Where("LOWER(tag_rule.repo_name) = LOWER(?) AND tag_rule.namespace =? AND tag_rule.repo_type = ? AND tag_rule.category = ?", repoName, namespace, repoType, category). + Limit(1). + Scan(ctx) + if err != nil { + return nil, err + } + return &TagRule, nil +} diff --git a/builder/store/database/tag_rule_test.go b/builder/store/database/tag_rule_test.go new file mode 100644 index 00000000..23db7982 --- /dev/null +++ b/builder/store/database/tag_rule_test.go @@ -0,0 +1,34 @@ +package database_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/tests" + "opencsg.com/csghub-server/common/types" +) + +func TestTagRuleStore_FindByRepo(t *testing.T) { + db := tests.InitTestDB() + defer db.Close() + ctx := context.TODO() + + store := database.NewTagRuleStoreWithDB(db) + + _, err := db.Core.NewInsert().Model(&database.TagRule{ + Category: "foo", + Namespace: "test", + RepoName: "bar", + RepoType: string(types.ModelRepo), + TagName: "t1", + }).Exec(ctx) + require.Nil(t, err) + tr, err := store.FindByRepo(ctx, "foo", "test", "bar", string(types.ModelRepo)) + require.Nil(t, err) + require.Equal(t, "t1", tr.TagName) + + _, err = store.FindByRepo(ctx, "foo", "test", "foo", string(types.ModelRepo)) + require.NotNil(t, err) +} diff --git a/cmd/csghub-server/cmd/deploy/runner.go b/cmd/csghub-server/cmd/deploy/runner.go index 3fa67b9d..a9b970fc 100644 --- a/cmd/csghub-server/cmd/deploy/runner.go +++ b/cmd/csghub-server/cmd/deploy/runner.go @@ -1,16 +1,19 @@ package deploy import ( + "fmt" + "github.com/spf13/cobra" "opencsg.com/csghub-server/api/httpbase" + "opencsg.com/csghub-server/builder/event" "opencsg.com/csghub-server/builder/store/database" "opencsg.com/csghub-server/common/config" - "opencsg.com/csghub-server/servicerunner/router" + "opencsg.com/csghub-server/runner/router" ) var startRunnerCmd = &cobra.Command{ Use: "runner", - Short: "start space runner service", + Short: "start runner service", PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) { return }, @@ -25,7 +28,10 @@ var startRunnerCmd = &cobra.Command{ DSN: config.Database.DSN, } database.InitDB(dbConfig) - + err = event.InitEventPublisher(config) + if err != nil { + return fmt.Errorf("fail to initialize message queue, %w", err) + } s, err := router.NewHttpServer(config) if err != nil { return err diff --git a/cmd/csghub-server/cmd/start/server.go b/cmd/csghub-server/cmd/start/server.go index a684674c..e66787af 100644 --- a/cmd/csghub-server/cmd/start/server.go +++ b/cmd/csghub-server/cmd/start/server.go @@ -78,6 +78,7 @@ var serverCmd = &cobra.Command{ ModelDownloadEndpoint: cfg.Model.DownloadEndpoint, PublicRootDomain: cfg.Space.PublicRootDomain, S3Internal: s3Internal, + IsMasterHost: cfg.IsMasterHost, }) if err != nil { return fmt.Errorf("failed to init deploy: %w", err) diff --git a/common/config/config.go b/common/config/config.go index 3018f1f9..7d85a642 100644 --- a/common/config/config.go +++ b/common/config/config.go @@ -18,6 +18,8 @@ type Config struct { // enable if you want to acess csghub through https, especially for space rproxy EnableHTTPS bool `env:"STARHUB_SERVER_ENABLE_HTTPS, default=false"` DocsHost string `env:"STARHUB_SERVER_SERVER_DOCS_HOST, default=http://localhost:6636"` + //the master host + IsMasterHost bool `env:"STARHUB_SERVER_IS_MASTER_HOST, default=true"` APIServer struct { Port int `env:"STARHUB_SERVER_SERVER_PORT, default=8080"` @@ -210,6 +212,18 @@ type Config struct { WorkFLow struct { Endpoint string `env:"OPENCSG_WORKFLOW_SERVER_ENDPOINT, default=localhost:7233"` } + + Argo struct { + Namespace string `env:"STARHUB_SERVER_ARGO_NAMESPACE, default=workflows"` + // NamespaceQuota is used to create evaluation with free of charge + QuotaNamespace string `env:"STARHUB_SERVER_ARGO_QUOTA_NAMESPACE, default=workflows-quota"` + QuotaGPUNumber string `env:"STARHUB_SERVER_ARGO_QUOTA_GPU_NUMBER, default=1"` + //job will be deleted after JobTTL seconds once the jobs was done + JobTTL int `env:"STARHUB_SERVER_ARGO_TTL, default=120"` + ServiceAccountName string `env:"STARHUB_SERVER_ARGO_SERVICE_ACCOUNT, default=executor"` + // S3PublicBucket is used to store public files, should set bucket same with portal + S3PublicBucket string `env:"STARHUB_SERVER_ARGO_S3_PUBLIC_BUCKET"` + } } func SetConfigFile(file string) { diff --git a/common/types/accounting.go b/common/types/accounting.go index 2856cc49..303f6a91 100644 --- a/common/types/accounting.go +++ b/common/types/accounting.go @@ -31,6 +31,7 @@ var ( SceneSpace SceneType = 11 // csghub space SceneModelFinetune SceneType = 12 // model finetune SceneMultiSync SceneType = 13 // multi sync + SceneEvaluation SceneType = 14 // model evaluation SceneStarship SceneType = 20 // starship SceneUnknow SceneType = 99 // unknow ) diff --git a/common/types/argo_workflow.go b/common/types/argo_workflow.go new file mode 100644 index 00000000..87931438 --- /dev/null +++ b/common/types/argo_workflow.go @@ -0,0 +1,157 @@ +package types + +import ( + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" +) + +var WorkFlowFinished = map[v1alpha1.WorkflowPhase]struct{}{ + v1alpha1.WorkflowSucceeded: {}, + v1alpha1.WorkflowFailed: {}, + v1alpha1.WorkflowError: {}, +} + +var _ SensitiveRequestV2 = (*EvaluationReq)(nil) + +func (c *EvaluationReq) GetSensitiveFields() []SensitiveField { + return []SensitiveField{ + { + Name: "task_name", + Value: func() string { + return c.TaskName + }, + Scenario: "nickname_detection", + }, + { + Name: "task_desc", + Value: func() string { + return c.TaskDesc + }, + Scenario: "comment_detection", + }, + } +} + +type TaskType string + +const ( + TaskTypeEvaluation TaskType = "evaluation" + TaskTypeTraining TaskType = "training" + TaskTypeComparison TaskType = "comparison" + TaskTypeLeaderBoard TaskType = "leaderboard" +) + +type EvaluationReq struct { + Username string `json:"-"` + TaskName string `json:"task_name"` + TaskDesc string `json:"task_desc"` + RuntimeFrameworkId int64 `json:"runtime_framework_id"` // ArgoWorkFlow framework + Datasets []string `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id"` + ModelId string `json:"model_id"` + ShareMode bool `json:"share_mode"` + Token string `json:"-"` + Hardware HardWare `json:"-"` + UserUUID string `json:"-"` + ClusterID string `json:"-"` + Image string `json:"-"` + RepoType string `json:"-"` + TaskType TaskType `json:"-"` + DownloadEndpoint string `json:"-"` + ResourceName string `json:"-"` +} + +type ArgoFlowTemplate struct { + Name string `json:"name"` + Image string `json:"image"` + Command []string `json:"command"` + Args []string `json:"args"` + HardWare HardWare `json:"hardware,omitempty"` + Env map[string]string `json:"env,omitempty"` + Annotation map[string]string `json:"annotation,omitempty"` +} + +type ArgoWorkFlowReq struct { + ClusterID string `json:"cluster_id"` + RepoType string `json:"repo_type"` + Templates []ArgoFlowTemplate `json:"templates,omitempty"` + Entrypoint string `json:"entrypoint"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + RepoIds []string `json:"repo_ids"` + TaskDesc string `json:"task_desc"` + Image string `json:"image"` // ArgoWorkFlow framework + Datasets []string `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id"` + ResourceName string `json:"resource_name"` + UserUUID string `json:"user_uuid"` + ShareMode bool `json:"share_mode"` +} + +type ArgoWorkFlowListRes struct { + List []ArgoWorkFlowRes `json:"list"` + Total int `json:"total"` +} + +type ArgoWorkFlowRes struct { + ID int64 `json:"id"` + RepoIds []string `json:"repo_ids"` + RepoType string `json:"repo_type,omitempty"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + TaskDesc string `json:"task_desc"` + Datasets []string `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id,omitempty"` + Status v1alpha1.WorkflowPhase `json:"status"` + Reason string `json:"reason,omitempty"` + Image string `bun:",notnull" json:"image"` + SubmitTime time.Time `json:"submit_time"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + ResultURL string `json:"result_url"` + DownloadURL string `json:"download_url"` + FailuresURL string `json:"failures_url"` +} + +type RepoTags struct { + RepoId string `json:"repo_id"` + Tags []RepoTag `json:"tags"` +} + +type EvaluationRes struct { + ID int64 `json:"id"` + RepoIds []string `json:"repo_ids"` + RepoType string `json:"repo_type,omitempty"` + Username string `json:"username"` + TaskName string `json:"task_name"` + TaskId string `json:"task_id"` + TaskType TaskType `json:"task_type"` + TaskDesc string `json:"task_desc"` + Datasets []RepoTags `json:"datasets,omitempty"` + ResourceId int64 `json:"resource_id,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Image string `bun:",notnull" json:"image"` + SubmitTime time.Time `json:"submit_time"` + StartTime time.Time `json:"start_time,omitempty"` + EndTime time.Time `json:"end_time,omitempty"` + ResultURL string `json:"result_url"` + DownloadURL string `json:"download_url"` + FailuresURL string `json:"failures_url"` +} + +type ( + EvaluationDelReq = ArgoWorkFlowDeleteReq + EvaluationGetReq = ArgoWorkFlowDeleteReq + ArgoWorkFlowGetReq = ArgoWorkFlowDeleteReq +) + +type ArgoWorkFlowDeleteReq struct { + ID int64 `json:"id"` + Username string `json:"username"` +} diff --git a/common/types/mirror.go b/common/types/mirror.go index f9e6cecc..294bed0c 100644 --- a/common/types/mirror.go +++ b/common/types/mirror.go @@ -99,9 +99,10 @@ const ( type Mapping string const ( - AutoMapping Mapping = "auto" - HFMapping Mapping = "hf" - CSGHubMapping Mapping = "csghub" + AutoMapping Mapping = "auto" + HFMapping Mapping = "hf" + CSGHubMapping Mapping = "csghub" + ModelScopeMapping Mapping = "ms" ) type MirrorPriority int diff --git a/common/types/model.go b/common/types/model.go index 263394a5..028aa4a1 100644 --- a/common/types/model.go +++ b/common/types/model.go @@ -154,18 +154,21 @@ type Model struct { // widget UI style: generation,chat WidgetType ModelWidgetType `json:"widget_type" example:"generation"` // url to interact with the model - Status string `json:"status" example:"RUNNING"` - UserLikes bool `json:"user_likes"` - Source RepositorySource `json:"source"` - SyncStatus RepositorySyncStatus `json:"sync_status"` - EnableInference bool `json:"enable_inference"` - EnableFinetune bool `json:"enable_finetune"` - BaseModel string `json:"base_model"` - License string `json:"license"` - CanWrite bool `json:"can_write"` - CanManage bool `json:"can_manage"` - Namespace *Namespace `json:"namespace"` - MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` + Status string `json:"status" example:"RUNNING"` + UserLikes bool `json:"user_likes"` + Source RepositorySource `json:"source"` + SyncStatus RepositorySyncStatus `json:"sync_status"` + EnableInference bool `json:"enable_inference"` + EnableFinetune bool `json:"enable_finetune"` + EnableEvaluation bool `json:"enable_evaluation"` + BaseModel string `json:"base_model"` + License string `json:"license"` + CanWrite bool `json:"can_write"` + CanManage bool `json:"can_manage"` + Namespace *Namespace `json:"namespace"` + RecomOpWeight int `json:"recom_op_weight,omitempty"` + SensitiveCheckStatus string `json:"sensitive_check_status"` + MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"` } type SDKModelInfo struct { @@ -275,6 +278,8 @@ const ( InferenceType = 1 // inference endpoint FinetuneType = 2 // finetune ServerlessType = 3 // serverless + EvaluationType = 4 // evaluation + UnknownType = -1 // unknown case ) type DeployActReq struct { diff --git a/common/types/tag.go b/common/types/tag.go index 847e0310..7e20074f 100644 --- a/common/types/tag.go +++ b/common/types/tag.go @@ -11,3 +11,14 @@ type RepoTag struct { CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } + +type TagCategory string + +const ( + TaskCategory TagCategory = "task" + LicenseCategory TagCategory = "license" + FrameworkCategory TagCategory = "framework" + SizeCategory TagCategory = "size" + LanguageCategory TagCategory = "language" + EvaluationCategory TagCategory = "evaluation" +) diff --git a/common/types/user.go b/common/types/user.go index 5c4b8b0f..18d4ab65 100644 --- a/common/types/user.go +++ b/common/types/user.go @@ -141,6 +141,7 @@ type ( UserCollectionReq = UserDatasetsReq DeleteUserTokenRequest = CreateUserTokenRequest UserPromptsReq = UserDatasetsReq + UserEvaluationReq = UserDatasetsReq ) type PageOpts struct { diff --git a/component/callback/git_callback.go b/component/callback/git_callback.go index 4c41798f..b1bddcd4 100644 --- a/component/callback/git_callback.go +++ b/component/callback/git_callback.go @@ -2,10 +2,12 @@ package callback import ( "context" + "database/sql" "errors" "fmt" "log/slog" "path" + "slices" "strconv" "strings" "time" @@ -37,6 +39,7 @@ type GitCallbackComponent struct { ras database.RuntimeArchitecturesStore rfs database.RuntimeFrameworksStore ts database.TagStore + dt database.TagRuleStore // set visibility if file content is sensitive setRepoVisibility bool pp component.PromptComponent @@ -79,6 +82,7 @@ func NewGitCallback(config *config.Config) (*GitCallbackComponent, error) { if config.SensitiveCheck.Enable { modSvcClient = rpc.NewModerationSvcHttpClient(fmt.Sprintf("%s:%d", config.Moderation.Host, config.Moderation.Port)) } + dt := database.NewTagRuleStore() return &GitCallbackComponent{ config: config, gs: gs, @@ -97,6 +101,7 @@ func NewGitCallback(config *config.Config) (*GitCallbackComponent, error) { rfs: rfs, pp: pp, ts: ts, + dt: dt, maxPromptFS: config.Dataset.PromptMaxJsonlFileSize, }, nil } @@ -211,7 +216,7 @@ func (c *GitCallbackComponent) modifyFiles(ctx context.Context, repoType, namesp for _, fileName := range fileNames { slog.Debug("modify file", slog.String("file", fileName)) // update model runtime - c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, false) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false, fileNames) // only care about readme file under root directory if fileName != types.ReadmeFileName { continue @@ -233,7 +238,7 @@ func (c *GitCallbackComponent) removeFiles(ctx context.Context, repoType, namesp for _, fileName := range fileNames { slog.Debug("remove file", slog.String("file", fileName)) // update model runtime - c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, true) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, true, fileNames) // only care about readme file under root directory if fileName == types.ReadmeFileName { // use empty content to clear all the meta tags @@ -278,7 +283,7 @@ func (c *GitCallbackComponent) addFiles(ctx context.Context, repoType, namespace for _, fileName := range fileNames { slog.Debug("add file", slog.String("file", fileName)) // update model runtime - c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, false) + c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false, fileNames) // only care about readme file under root directory if fileName == types.ReadmeFileName { content, err := c.getFileRaw(repoType, namespace, repoName, ref, fileName) @@ -374,10 +379,73 @@ func (c *GitCallbackComponent) getFileRaw(repoType, namespace, repoName, ref, fi return content, nil } -func (c *GitCallbackComponent) updateModelRuntimeFrameworks(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) { +// update repo relations +func (c *GitCallbackComponent) updateRepoRelations(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool, fileNames []string) { slog.Debug("update model relation for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("repoType", repoType), slog.Any("fileName", fileName), slog.Any("branch", ref)) + if repoType == fmt.Sprintf("%ss", types.ModelRepo) { + c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, deleteAction) + } + if repoType == fmt.Sprintf("%ss", types.DatasetRepo) { + c.updateDatasetTags(ctx, namespace, repoName, fileNames) + } +} + +// update dataset tags for evaluation +func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, repoName string, fileNames []string) { + // script dataset repo was not supported so far + scriptName := fmt.Sprintf("%s.py", repoName) + if slices.Contains(fileNames, scriptName) { + return + } + repo, err := c.rs.FindByPath(ctx, types.DatasetRepo, namespace, repoName) + if err != nil || repo == nil { + slog.Warn("fail to query repo for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) + return + } + // check if it's evaluation dataset + evalDataset, err := c.dt.FindByRepo(ctx, string(types.EvaluationCategory), namespace, repoName, string(types.DatasetRepo)) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + // check if it's a mirror repo + mirror, err := c.mirrorStore.FindByRepoPath(ctx, types.DatasetRepo, namespace, repoName) + if err != nil || mirror == nil { + slog.Debug("fail to query mirror dataset for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err)) + return + } + namespace := strings.Split(mirror.SourceRepoPath, "/")[0] + name := strings.Split(mirror.SourceRepoPath, "/")[1] + // use mirror namespace and name to find dataset + evalDataset, err = c.dt.FindByRepo(ctx, string(types.EvaluationCategory), namespace, name, string(types.DatasetRepo)) + if err != nil { + slog.Debug("not an evaluation dataset, ignore it", slog.Any("repo id", repo.Path)) + return + } + } else { + slog.Error("failed to query evaluation dataset", slog.Any("repo id", repo.Path), slog.Any("error", err)) + return + } + + } + tagIds := []int64{} + tagIds = append(tagIds, evalDataset.Tag.ID) + if evalDataset.RuntimeFramework != "" { + rTag, _ := c.ts.FindTag(ctx, evalDataset.RuntimeFramework, string(types.DatasetRepo), "runtime_framework") + if rTag != nil { + tagIds = append(tagIds, rTag.ID) + } + } + + err = c.ts.UpsertRepoTags(ctx, repo.ID, []int64{}, tagIds) + if err != nil { + slog.Warn("fail to add dataset tag", slog.Any("repoId", repo.ID), slog.Any("tag id", tagIds), slog.Any("error", err)) + } + +} + +// update model runtime frameworks +func (c *GitCallbackComponent) updateModelRuntimeFrameworks(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) { // must be model repo and config.json - if repoType != fmt.Sprintf("%ss", types.ModelRepo) || fileName != component.ConfigFileName || ref != ("refs/heads/"+component.MainBranch) { + if repoType != fmt.Sprintf("%ss", types.ModelRepo) || fileName != component.ConfigFileName || (ref != ("refs/heads/"+component.MainBranch) && ref != ("refs/heads/"+component.MasterBranch)) { return } repo, err := c.rs.FindByPath(ctx, types.ModelRepo, namespace, repoName) diff --git a/component/evaluation.go b/component/evaluation.go new file mode 100644 index 00000000..683261cb --- /dev/null +++ b/component/evaluation.go @@ -0,0 +1,191 @@ +package component + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + + "opencsg.com/csghub-server/builder/deploy" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" +) + +type evaluationComponentImpl struct { + deployer deploy.Deployer + userStore database.UserStore + modelStore database.ModelStore + datasetStore database.DatasetStore + mirrorStore database.MirrorStore + spaceResourceStore database.SpaceResourceStore + tokenStore database.AccessTokenStore + rtfm database.RuntimeFrameworksStore + config *config.Config + ac AccountingComponent +} + +type EvaluationComponent interface { + // Create argo workflow + CreateEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) + GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) + DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error +} + +func NewEvaluationComponent(config *config.Config) (EvaluationComponent, error) { + c := &evaluationComponentImpl{} + c.deployer = deploy.NewDeployer() + c.userStore = database.NewUserStore() + c.modelStore = database.NewModelStore() + c.spaceResourceStore = database.NewSpaceResourceStore() + c.datasetStore = database.NewDatasetStore() + c.mirrorStore = database.NewMirrorStore() + c.tokenStore = database.NewAccessTokenStore() + c.rtfm = database.NewRuntimeFrameworksStore() + c.config = config + ac, err := NewAccountingComponent(config) + if err != nil { + return nil, fmt.Errorf("failed to create accounting component, %w", err) + } + c.ac = ac + return c, nil +} + +// Create argo workflow +func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) { + user, err := c.userStore.FindByUsername(ctx, req.Username) + if err != nil { + return nil, fmt.Errorf("failed to get current user %s, error:%w", req.Username, err) + } + result := strings.Split(req.ModelId, "/") + _, err = c.modelStore.FindByPath(ctx, result[0], result[1]) + if err != nil { + return nil, fmt.Errorf("cannot find model, %w", err) + } + + token, err := c.tokenStore.FindByUID(ctx, user.ID) + if err != nil { + return nil, fmt.Errorf("cant get git access token:%w", err) + } + mirrorRepos, err := c.GenerateMirrorRepoIds(ctx, req.Datasets) + if err != nil { + return nil, fmt.Errorf("failed to generate mirror repo ids, %w", err) + } + req.Datasets = mirrorRepos + req.Token = token.Token + var hardware types.HardWare + if req.ResourceId != 0 { + resource, err := c.spaceResourceStore.FindByID(ctx, req.ResourceId) + if err != nil { + return nil, fmt.Errorf("cannot find resource, %w", err) + } + err = json.Unmarshal([]byte(resource.Resources), &hardware) + if err != nil { + return nil, fmt.Errorf("invalid hardware setting, %w", err) + } + if hardware.Gpu.Num == "" { + return nil, fmt.Errorf("evaluation requires GPU or NPU resources") + } + req.ClusterID = resource.ClusterID + req.ResourceName = resource.Name + } else { + // for share mode + hardware.Gpu.Num = c.config.Argo.QuotaGPUNumber + hardware.Gpu.ResourceName = "nvidia.com/gpu" + hardware.Cpu.Num = "8" + hardware.Memory = "32Gi" + } + frame, err := c.rtfm.FindEnabledByID(ctx, req.RuntimeFrameworkId) + if err != nil { + return nil, fmt.Errorf("cannot find available runtime framework, %w", err) + } + req.Hardware = hardware + // choose image + containerImg := frame.FrameImage + req.UserUUID = user.UUID + req.Image = containerImg + req.RepoType = string(types.ModelRepo) + req.TaskType = types.TaskTypeEvaluation + req.DownloadEndpoint = c.config.Model.DownloadEndpoint + return c.deployer.SubmitEvaluation(ctx, req) +} + +// generate mirror repo ids +func (c *evaluationComponentImpl) GenerateMirrorRepoIds(ctx context.Context, datasets []string) ([]string, error) { + var mirrorRepos []string + for _, ds := range datasets { + namespace := strings.Split(ds, "/")[0] + name := strings.Split(ds, "/")[1] + mirrorRepo, err := c.mirrorStore.FindByRepoPath(ctx, types.DatasetRepo, namespace, name) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + //no mirror, will use csghub repo + mirrorRepos = append(mirrorRepos, ds) + continue + } + return nil, fmt.Errorf("fail to get mirror repo, %w", err) + } + mirrorRepos = append(mirrorRepos, mirrorRepo.SourceRepoPath) + } + return mirrorRepos, nil +} + +func (c *evaluationComponentImpl) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error { + return c.deployer.DeleteEvaluation(ctx, req) +} + +// get evaluation result +func (c *evaluationComponentImpl) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) { + wf, err := c.deployer.GetEvaluation(ctx, req) + if err != nil { + return nil, fmt.Errorf("fail to get evaluation result, %w", err) + } + datasets, err := c.datasetStore.ListByPath(ctx, wf.Datasets) + if err != nil { + return nil, fmt.Errorf("fail to get datasets for evaluation, %w", err) + } + var repoTags []types.RepoTags + for _, ds := range datasets { + var tags []types.RepoTag + for _, tag := range ds.Repository.Tags { + tags = append(tags, types.RepoTag{ + Name: tag.Name, + Category: tag.Category, + Group: tag.Group, + BuiltIn: tag.BuiltIn, + ShowName: tag.ShowName, + CreatedAt: tag.CreatedAt, + UpdatedAt: tag.UpdatedAt, + }) + } + var dsRepoTags = types.RepoTags{ + RepoId: ds.Repository.Path, + Tags: tags, + } + repoTags = append(repoTags, dsRepoTags) + } + var res = &types.EvaluationRes{ + ID: wf.ID, + RepoIds: wf.RepoIds, + RepoType: wf.RepoType, + Username: wf.Username, + TaskName: wf.TaskName, + TaskId: wf.TaskId, + TaskType: wf.TaskType, + TaskDesc: wf.TaskDesc, + ResourceId: wf.ResourceId, + Status: string(wf.Status), + Reason: wf.Reason, + Datasets: repoTags, + Image: wf.Image, + SubmitTime: wf.SubmitTime, + StartTime: wf.StartTime, + EndTime: wf.EndTime, + ResultURL: wf.ResultURL, + DownloadURL: wf.DownloadURL, + FailuresURL: wf.FailuresURL, + } + return res, nil +} diff --git a/component/evaluation_test.go b/component/evaluation_test.go new file mode 100644 index 00000000..a1620f14 --- /dev/null +++ b/component/evaluation_test.go @@ -0,0 +1,198 @@ +package component + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + mock_deploy "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/deploy" + mock_component "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component" + "opencsg.com/csghub-server/builder/deploy" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/tests" + "opencsg.com/csghub-server/common/types" +) + +func NewTestEvaluationComponent(deployer deploy.Deployer, stores *tests.MockStores, ac AccountingComponent) EvaluationComponent { + cfg := &config.Config{} + cfg.Argo.QuotaGPUNumber = "1" + return &evaluationComponentImpl{ + deployer: deployer, + config: cfg, + userStore: stores.User, + modelStore: stores.Model, + datasetStore: stores.Dataset, + mirrorStore: stores.Mirror, + spaceResourceStore: stores.SpaceResource, + tokenStore: stores.AccessToken, + rtfm: stores.RuntimeFramework, + ac: ac, + } +} + +func TestEvaluationComponent_CreateEvaluation(t *testing.T) { + req := types.EvaluationReq{ + TaskName: "test", + ModelId: "opencsg/wukong", + Username: "test", + ResourceId: 0, + Datasets: []string{"opencsg/hellaswag"}, + RuntimeFrameworkId: 1, + } + ctx := context.TODO() + req2 := types.EvaluationReq{ + UserUUID: "test", + TaskName: "test", + ModelId: "opencsg/wukong", + Username: "test", + ResourceId: 0, + Datasets: []string{"Rowan/hellaswag"}, + RuntimeFrameworkId: 1, + Hardware: types.HardWare{ + Gpu: types.GPU{ + Num: "1", + ResourceName: "nvidia.com/gpu", + }, + Cpu: types.CPU{ + Num: "8", + }, + Memory: "32Gi", + }, + Image: "lm-evaluation-harness:0.4.6", + RepoType: "model", + TaskType: "evaluation", + Token: "foo", + } + t.Run("create evaluation without resource id", func(t *testing.T) { + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + stores.UserMock().EXPECT().FindByUsername(ctx, req.Username).Return(database.User{ + RoleMask: "admin", + Username: req.Username, + UUID: req.Username, + ID: 1, + }, nil).Once() + stores.ModelMock().EXPECT().FindByPath(ctx, "opencsg", "wukong").Return( + &database.Model{ + ID: 1, + }, nil, + ).Maybe() + stores.MirrorMock().EXPECT().FindByRepoPath(ctx, types.DatasetRepo, "opencsg", "hellaswag").Return(&database.Mirror{ + SourceRepoPath: "Rowan/hellaswag", + }, nil) + stores.AccessTokenMock().EXPECT().FindByUID(ctx, int64(1)).Return(&database.AccessToken{Token: "foo"}, nil) + stores.RuntimeFrameworkMock().EXPECT().FindEnabledByID(ctx, int64(1)).Return(&database.RuntimeFramework{ + ID: 1, + FrameImage: "lm-evaluation-harness:0.4.6", + }, nil) + deployerMock.EXPECT().SubmitEvaluation(ctx, req2).Return(&types.ArgoWorkFlowRes{ + ID: 1, + TaskName: "test", + }, nil) + e, err := c.CreateEvaluation(ctx, req) + require.NotNil(t, e) + require.Equal(t, "test", e.TaskName) + require.Nil(t, err) + }) + t.Run("create evaluation with resource id", func(t *testing.T) { + req.ResourceId = 1 + req2.ResourceId = 1 + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + stores.UserMock().EXPECT().FindByUsername(ctx, req.Username).Return(database.User{ + RoleMask: "admin", + Username: req.Username, + UUID: req.Username, + ID: 1, + }, nil).Once() + stores.ModelMock().EXPECT().FindByPath(ctx, "opencsg", "wukong").Return( + &database.Model{ + ID: 1, + }, nil, + ).Maybe() + stores.MirrorMock().EXPECT().FindByRepoPath(ctx, types.DatasetRepo, "opencsg", "hellaswag").Return(&database.Mirror{ + SourceRepoPath: "Rowan/hellaswag", + }, nil) + stores.AccessTokenMock().EXPECT().FindByUID(ctx, int64(1)).Return(&database.AccessToken{Token: "foo"}, nil) + stores.RuntimeFrameworkMock().EXPECT().FindEnabledByID(ctx, int64(1)).Return(&database.RuntimeFramework{ + ID: 1, + FrameImage: "lm-evaluation-harness:0.4.6", + }, nil) + resource, err := json.Marshal(req2.Hardware) + require.Nil(t, err) + stores.SpaceResourceMock().EXPECT().FindByID(ctx, int64(1)).Return(&database.SpaceResource{ + ID: 1, + Resources: string(resource), + }, nil) + deployerMock.EXPECT().SubmitEvaluation(ctx, req2).Return(&types.ArgoWorkFlowRes{ + ID: 1, + TaskName: "test", + }, nil) + e, err := c.CreateEvaluation(ctx, req) + require.NotNil(t, e) + require.Equal(t, "test", e.TaskName) + require.Nil(t, err) + }) +} + +func TestEvaluationComponent_GetEvaluation(t *testing.T) { + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + req := types.EvaluationGetReq{ + Username: "test", + } + ctx := context.TODO() + deployerMock.EXPECT().GetEvaluation(ctx, req).Return(&types.ArgoWorkFlowRes{ + ID: 1, + RepoIds: []string{"Rowan/hellaswag"}, + Datasets: []string{"Rowan/hellaswag"}, + RepoType: "model", + Username: "test", + TaskName: "test", + TaskId: "test", + TaskType: "evaluation", + Status: "Succeed", + }, nil) + stores.DatasetMock().EXPECT().ListByPath(ctx, []string{"Rowan/hellaswag"}).Return([]database.Dataset{ + { + Repository: &database.Repository{ + Path: "Rowan/hellaswag", + Tags: []database.Tag{ + { + Name: "test", + Category: "test", + Group: "test", + Scope: "test", + BuiltIn: true, + }, + }, + }, + }, + }, nil) + e, err := c.GetEvaluation(ctx, req) + require.NotNil(t, e) + require.Equal(t, "test", e.TaskName) + require.Nil(t, err) +} + +func TestEvaluationComponent_DeleteEvaluation(t *testing.T) { + deployerMock := &mock_deploy.MockDeployer{} + stores := tests.NewMockStores(t) + ac := &mock_component.MockAccountingComponent{} + c := NewTestEvaluationComponent(deployerMock, stores, ac) + req := types.EvaluationDelReq{ + Username: "test", + } + ctx := context.TODO() + deployerMock.EXPECT().DeleteEvaluation(ctx, req).Return(nil) + err := c.DeleteEvaluation(ctx, req) + require.Nil(t, err) +} diff --git a/component/mirror.go b/component/mirror.go index 04f1cec6..050ae92e 100644 --- a/component/mirror.go +++ b/component/mirror.go @@ -332,6 +332,7 @@ var mirrorOrganizationMap = map[string]string{ "openbmb": "OpenBMB", "netease-youdao": "Netease-youdao", "ByteDance": "ByteDance", + "opencompass": "opencompass", } var mirrorStatusAndRepoSyncStatusMapping = map[types.MirrorTaskStatus]types.RepositorySyncStatus{ diff --git a/component/model.go b/component/model.go index 937fa64d..c06c6e80 100644 --- a/component/model.go +++ b/component/model.go @@ -476,6 +476,10 @@ func (c *modelComponentImpl) Show(ctx context.Context, namespace, name, currentU if len(finetunes) > 0 { resModel.EnableFinetune = true } + evaluations, _ := c.repoRuntimeFrameworkStore.GetByRepoIDsAndType(ctx, model.Repository.ID, types.EvaluationType) + if len(evaluations) > 0 { + resModel.EnableEvaluation = true + } return resModel, nil } @@ -1129,17 +1133,19 @@ func (c *modelComponentImpl) ListModelsOfRuntimeFrameworks(ctx context.Context, // define EnableInference enableInference := deployType == types.InferenceType enableFinetune := deployType == types.FinetuneType + enableEvaluation := deployType == types.EvaluationType for _, repo := range repos { resModels = append(resModels, types.Model{ - Name: repo.Name, - Nickname: repo.Nickname, - Description: repo.Description, - Path: repo.Path, - RepositoryID: repo.ID, - Private: repo.Private, - EnableInference: enableInference, - EnableFinetune: enableFinetune, + Name: repo.Name, + Nickname: repo.Nickname, + Description: repo.Description, + Path: repo.Path, + RepositoryID: repo.ID, + Private: repo.Private, + EnableInference: enableInference, + EnableFinetune: enableFinetune, + EnableEvaluation: enableEvaluation, }) } return resModels, total, nil diff --git a/component/model_test.go b/component/model_test.go index f90bc1d5..ea557131 100644 --- a/component/model_test.go +++ b/component/model_test.go @@ -232,9 +232,10 @@ func TestModelComponent_Show(t *testing.T) { HTTPCloneURL: "https://foo.com/s/foo/bar.git", SSHCloneURL: "test@127.0.0.1:s/foo/bar.git", }, - EnableInference: true, - EnableFinetune: true, - WidgetType: types.ModelWidgetTypeGeneration, + EnableInference: true, + EnableFinetune: true, + EnableEvaluation: true, + WidgetType: types.ModelWidgetTypeGeneration, }, model) } @@ -571,25 +572,22 @@ func TestModelComponent_SetRuntimeFrameworkModes(t *testing.T) { mc := initializeTestModelComponent(ctx, t) mc.mocks.stores.RuntimeFrameworkMock().EXPECT().FindByID(ctx, int64(1)).Return( - &database.RuntimeFramework{}, nil, + &database.RuntimeFramework{ + ID: 1, + }, nil, ) mc.mocks.stores.ModelMock().EXPECT().ListByPath(ctx, []string{"a", "b"}).Return( []database.Model{ {RepositoryID: 1, Repository: &database.Repository{ID: 1, Path: "m1/foo"}}, - {RepositoryID: 2, Repository: &database.Repository{ID: 2, Path: "m2/foo"}}, }, nil, ) rftags := []*database.Tag{{Name: "t1"}, {Name: "t2"}} mc.mocks.stores.TagMock().EXPECT().GetTagsByScopeAndCategories( ctx, database.TagScope("model"), []string{"runtime_framework", "resource"}, ).Return(rftags, nil) - mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().GetByIDsAndType( ctx, int64(1), int64(1), 1, - ).Return([]database.RepositoriesRuntimeFramework{}, nil) - mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().GetByIDsAndType( - ctx, int64(1), int64(2), 1, - ).Return([]database.RepositoriesRuntimeFramework{{}}, nil) + ).Return(nil, nil) mc.mocks.stores.RepoRuntimeFrameworkMock().EXPECT().Add(ctx, int64(1), int64(1), 1).Return(nil) mc.mocks.components.runtimeArchitecture.EXPECT().AddRuntimeFrameworkTag( diff --git a/component/runtime_architecture.go b/component/runtime_architecture.go index 138adce0..cb79f139 100644 --- a/component/runtime_architecture.go +++ b/component/runtime_architecture.go @@ -16,6 +16,7 @@ import ( var ( MainBranch string = "main" + MasterBranch string = "master" ConfigFileName string = "config.json" ScanLock sync.Mutex ) diff --git a/component/tag.go b/component/tag.go index 495a4483..d428b895 100644 --- a/component/tag.go +++ b/component/tag.go @@ -15,6 +15,7 @@ import ( ) type TagComponent interface { + AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) AllTags(ctx context.Context) ([]database.Tag, error) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string) error UpdateMetaTags(ctx context.Context, tagScope database.TagScope, namespace, name, content string) ([]*database.RepositoryTag, error) @@ -38,9 +39,12 @@ type tagComponentImpl struct { sensitiveChecker rpc.ModerationSvcClient } -func (tc *tagComponentImpl) AllTags(ctx context.Context) ([]database.Tag, error) { +func (c *tagComponentImpl) AllTags(ctx context.Context) ([]database.Tag, error) { // TODO: query cache for tags at first - return tc.ts.AllTags(ctx) + return c.ts.AllTags(ctx) +} +func (c *tagComponentImpl) AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) { + return c.ts.AllTagsByScopeAndCategory(ctx, database.TagScope(scope), category) } func (c *tagComponentImpl) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string) error { diff --git a/component/user.go b/component/user.go index 544af650..86cc31e9 100644 --- a/component/user.go +++ b/component/user.go @@ -38,6 +38,7 @@ type UserComponent interface { ListServerless(ctx context.Context, req types.DeployReq) ([]types.DeployRepo, int, error) GetUserByName(ctx context.Context, userName string) (*database.User, error) Prompts(ctx context.Context, req *types.UserPromptsReq) ([]types.PromptRes, int, error) + Evaluations(ctx context.Context, req *types.UserEvaluationReq) ([]types.ArgoWorkFlowRes, int, error) } func NewUserComponent(config *config.Config) (UserComponent, error) { @@ -74,6 +75,8 @@ func NewUserComponent(config *config.Config) (UserComponent, error) { return nil, err } c.promptStore = database.NewPromptStore() + c.promptStore = database.NewPromptStore() + c.wfs = database.NewArgoWorkFlowStore() return c, nil } @@ -96,6 +99,7 @@ type userComponentImpl struct { // srs database.SpaceResourceStore // urs *database.UserResourcesStore promptStore database.PromptStore + wfs database.ArgoWorkFlowStore } func (c *userComponentImpl) Datasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error) { @@ -736,3 +740,20 @@ func (c *userComponentImpl) Prompts(ctx context.Context, req *types.UserPromptsR return resPrompts, total, nil } + +func (c *userComponentImpl) Evaluations(ctx context.Context, req *types.UserEvaluationReq) ([]types.ArgoWorkFlowRes, int, error) { + + _, err := c.userStore.FindByUsername(ctx, req.CurrentUser) + if err != nil { + newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err) + return nil, 0, newError + } + + res, err := c.deployer.ListEvaluations(ctx, req.CurrentUser, req.PageSize, req.Page) + if err != nil { + newError := fmt.Errorf("failed to get user evaluations,error:%w", err) + slog.Error(newError.Error()) + return nil, 0, newError + } + return res.List, res.Total, nil +} diff --git a/docker/evaluation/Dockerfile.lm-evaluation-harness b/docker/evaluation/Dockerfile.lm-evaluation-harness new file mode 100644 index 00000000..7ef795d8 --- /dev/null +++ b/docker/evaluation/Dockerfile.lm-evaluation-harness @@ -0,0 +1,17 @@ +FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 +RUN apt-get update && apt-get -y install git python3.10 python3-pip dumb-init \ + && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN pip install --upgrade pip && pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install --no-cache-dir accelerate transformers==4.46.3 \ + minio oss2 langdetect huggingface_hub openpyxl +WORKDIR /workspace/ +RUN git clone --depth 1 https://github.com/EleutherAI/lm-evaluation-harness.git --branch v0.4.6 --single-branch && \ + cd lm-evaluation-harness && pip install setuptools --upgrade --no-cache-dir -e \ + ".[ifeval,math,multilingual,sentencepiece]" +COPY ./lm-evaluation-harness/ /etc/csghub/ +RUN ln -s /usr/bin/python3 /usr/bin/python &&\ + chmod +x /etc/csghub/*.sh +ENV HUGGINGFACE_HUB_CACHE=/workspace/ \ + HF_HUB_ENABLE_HF_TRANSFER=0 +ENTRYPOINT [ "/usr/bin/dumb-init", "--" ] +CMD ["/etc/csghub/start.sh"] diff --git a/docker/evaluation/Dockerfile.opencompass b/docker/evaluation/Dockerfile.opencompass new file mode 100644 index 00000000..e4a59551 --- /dev/null +++ b/docker/evaluation/Dockerfile.opencompass @@ -0,0 +1,15 @@ +FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 +RUN apt-get update && apt-get -y install python3.10 python3-pip dumb-init \ + && apt-get clean && rm -rf /var/lib/apt/lists/* +RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + && pip install --no-cache-dir opencompass==0.3.5 \ + csghub-sdk==0.4.4 minio oss2 +RUN pip install --no-cache-dir vllm==0.6.3.post1 openpyxl +COPY ./opencompass/ /etc/csghub/ +RUN ln -s /usr/bin/python3 /usr/bin/python &&\ + chmod +x /etc/csghub/*.sh +WORKDIR /workspace/ +ENV HUGGINGFACE_HUB_CACHE=/workspace/ \ + HF_HUB_ENABLE_HF_TRANSFER=0 +ENTRYPOINT [ "/usr/bin/dumb-init", "--" ] +CMD ["/etc/csghub/start.sh"] diff --git a/docker/evaluation/README.md b/docker/evaluation/README.md new file mode 100644 index 00000000..cf271903 --- /dev/null +++ b/docker/evaluation/README.md @@ -0,0 +1,66 @@ +# LLM evalution docker image + +## Login Container Registry +```bash +OPENCSG_ACR="opencsg-registry.cn-beijing.cr.aliyuncs.com" +OPENCSG_ACR_USERNAME="" +OPENCSG_ACR_PASSWORD="" +echo "$OPENCSG_ACR_PASSWORD" | docker login $OPENCSG_ACR -u $OPENCSG_ACR_USERNAME --password-stdin +``` + +## Build Multi-Platform Images +```bash +export BUILDX_NO_DEFAULT_ATTESTATIONS=1 +export IMAGE_TAG=0.3.5 +docker buildx build --platform linux/amd64,linux/arm64 \ + -t ${OPENCSG_ACR}/public/opencompass:${IMAGE_TAG} \ + -t ${OPENCSG_ACR}/public/opencompass:latest \ + -f Dockerfile.opencompass \ + --push . +export IMAGE_TAG=0.4.6 +docker buildx build --platform linux/amd64,linux/arm64 \ + -t ${OPENCSG_ACR}/public/lm-evaluation-harness:${IMAGE_TAG} \ + -t ${OPENCSG_ACR}/public/lm-evaluation-harness:latest \ + -f Dockerfile.lm-evaluation-harness \ + --push . +``` +*The above command will create `linux/amd64` and `linux/arm64` images with the tags `${IMAGE_TAG}` and `latest` at the same time.* + +## Test the opencompass Image +```bash +docker run \ + -e ACCESS_TOKEN=xxxx \ + -e MODEL_ID="OpenCSG/csg-wukong-1B" \ + -e DATASET_IDS="xzgan/hellaswag" \ + -e HF_ENDPOINT=https://hub.opencsg.com \ + -e ASCEND_VISIBLE_DEVICES=7 \ + -e S3_ACCESS_ID="xxxx" \ + -e S3_ACCESS_SECRET="xxxx" \ + -e S3_BUCKET="xxxxx" \ + -e S3_ENDPOINT="xxxxx" \ + -e S3_SSL_ENABLED="true" \ + ${OPENCSG_ACR}/public/opencompass:${IMAGE_TAG} +``` + +## Test the lm-evaluation-harness Image +```bash +export IMAGE_TAG=0.4.6 +docker run \ + --gpus device=7 \ + -e ACCESS_TOKEN=xxxx \ + -e MODEL_ID="OpenCSG/csg-wukong-1B" \ + -e DATASET_IDS="Rowan/hellaswag" \ + -e HF_ENDPOINT=https://hub.opencsg.com\ + -e S3_ACCESS_ID="xxx" \ + -e S3_ACCESS_SECRET="xxx" \ + -e S3_BUCKET="xxx" \ + -e S3_ENDPOINT="xxx" \ + -e S3_SSL_ENABLED="true" \ + ${OPENCSG_ACR}/public/lm-evaluation-harness:${IMAGE_TAG} +``` + +## evaluation image name, version and cuda version +| Latest Image | Version | CUDA Version | +| --- | --- | --- | +| opencompass | 0.3.5 | 12.4 | +| lm-evaluation-harness | 0.4.6 | 12.4 | diff --git a/docker/evaluation/lm-evaluation-harness/download.py b/docker/evaluation/lm-evaluation-harness/download.py new file mode 100644 index 00000000..46501c03 --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/download.py @@ -0,0 +1,27 @@ +import os +import argparse +import datasets +# Load model directly +from huggingface_hub import snapshot_download + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Download repo.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_a = subparsers.add_parser('datasets', help='download datasets') + parser_a.add_argument('--dataset_ids', type=str, help='repo id') + parser_b = subparsers.add_parser('models', help='download model') + parser_b.add_argument('--model_ids', type=str, help='repo id') + + args = parser.parse_args() + + # split repo ids + + if args.command == 'models': + repo_ids = args.model_ids.split(',') + for repo_id in repo_ids: + snapshot_download(repo_id=repo_id) + elif args.command == 'datasets': + repo_ids = args.dataset_ids.split(',') + for repo_id in repo_ids: + snapshot_download(repo_id=repo_id, repo_type="dataset") diff --git a/docker/evaluation/lm-evaluation-harness/get_task.py b/docker/evaluation/lm-evaluation-harness/get_task.py new file mode 100644 index 00000000..ebca8865 --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/get_task.py @@ -0,0 +1,112 @@ +import collections +import argparse +import os + +from lm_eval import utils + + +def _config_is_task(config) -> bool: + if ("task" in config) and isinstance(config["task"], str): + return True + return False + + +def _config_is_group(config) -> bool: + if ("task" in config) and isinstance(config["task"], list): + return True + return False + + +def _config_is_python_task(config) -> bool: + if "class" in config: + return True + return False + + +def get_task_and_group(task_dir: str): + def _populate_tags_and_groups(config, task, tasks_and_groups, print_info): + # TODO: remove group in next release + if "tag" in config: + attr_list = config["tag"] + if isinstance(attr_list, str): + attr_list = [attr_list] + + for tag in attr_list: + if tag not in tasks_and_groups: + tasks_and_groups[tag] = { + "type": "tag", + "task": [task], + "yaml_path": -1, + } + elif tasks_and_groups[tag]["type"] != "tag": + break + else: + tasks_and_groups[tag]["task"].append(task) + + # TODO: remove group in next release + print_info = True + ignore_dirs = [ + "__pycache__", + ".ipynb_checkpoints", + ] + tasks_and_groups = collections.defaultdict() + for root, dirs, file_list in os.walk(task_dir): + dirs[:] = [d for d in dirs if d not in ignore_dirs] + for f in file_list: + if f.endswith(".yaml"): + yaml_path = os.path.join(root, f) + config = utils.load_yaml_config(yaml_path, mode="simple") + if _config_is_python_task(config): + # This is a python class config + task = config["task"] + tasks_and_groups[task] = { + "type": "python_task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + elif _config_is_group(config): + tasks_and_groups[config["group"]] = { + "type": "group", + "task": -1, + "yaml_path": yaml_path, + } + + elif _config_is_task(config): + # This is a task config + task = config["task"] + tasks_and_groups[task] = { + "type": "task", + "yaml_path": yaml_path, + } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) + + return tasks_and_groups + + +def get_related_task(task_dir): + tasks_and_groups = get_task_and_group(task_dir) + # loop group + all_groups = [] + all_tasks = [] + for task in tasks_and_groups: + if tasks_and_groups[task]["type"] == "group": + all_groups.append(task) + if tasks_and_groups[task]["type"] == "task": + all_tasks.append(task) + if len(all_groups) > 0: + sorted_tasks = sorted(all_groups, key=len) + return sorted_tasks[0] + else: + return ", ".join(all_tasks) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get dataset task by dir.') + parser.add_argument('task_dir', type=str, help='The task dir') + args = parser.parse_args() + + print(get_related_task(args.task_dir)) diff --git a/docker/evaluation/lm-evaluation-harness/start.sh b/docker/evaluation/lm-evaluation-harness/start.sh new file mode 100644 index 00000000..7d687d16 --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/start.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +search_path_with_most_term() { + search_term=$1 + paths=("${@:2}") + + declare -A count_dict + for path in "${paths[@]}"; do + count_dict["$path"]=$(grep -o "$search_term" "$path" | wc -l) + done + + max_count_path="" + max_count=0 + for path in "${!count_dict[@]}"; do + count=${count_dict[$path]} + if [[ $count -gt $max_count ]]; then + max_count=$count + max_count_path=$path + fi + done + if [ -z "$max_count_path" ]; then + echo $paths[0] + return 0 + fi + echo $max_count_path + return 0 +} +#download datasets +if [ ! -z "$DATASET_IDS" ]; then + echo "Downloading datasets..." + python /etc/csghub/download.py datasets --dataset_ids $DATASET_IDS +fi +if [ $? -ne 0 ]; then + echo "Failed to download datasets" + exit 1 +fi +#download models +if [ ! -z "$MODEL_ID" ]; then + echo "Downloading models..." + python /etc/csghub/download.py models --model_ids $MODEL_ID +fi +if [ $? -ne 0 ]; then + echo "Failed to download models" + exit 1 +fi + +export HF_ENDPOINT="$HF_ENDPOINT/hf" + +tasks="" +task_dir="/workspace/lm-evaluation-harness/lm_eval/tasks" +IFS=',' read -r -a dataset_repos <<< "$DATASET_IDS" +if [ -z "$NUM_FEW_SHOT" ]; then + NUM_FEW_SHOT=0 +fi +script_dts_array=("allenai/winogrande" "facebook/anli" "aps/super_glue" "Rowan/hellaswag" "nyu-mll/blimp" "EdinburghNLP/orange_sum" "facebook/xnli" "nyu-mll/glue" "openai/gsm8k" "cimec/lambada" "allenai/math_qa" "openlifescienceai/medmcqa" "google-research-datasets/nq_open" "allenai/openbookqa" "google-research-datasets/paws-x" "ybisk/piqa" "community-datasets/qa4mre" "allenai/sciq" "allenai/social_i_qa" "LSDSem/story_cloze" "allenai/swag" "IWSLT/iwslt2017" "wmt/wmt14" "wmt/wmt16","mandarjoshi/trivia_qa" "truthfulqa/truthful_qa" "Stanford/web_questions" "ErnestSDavis/winograd_wsc" "cambridgeltl/xcopa" "google/xquad") +for repo in "${dataset_repos[@]}"; do + repo_name="${repo#*/}" + if [[ " ${script_dts_array[@]} " =~ " ${repo} " ]]; then + #need replace with real path + echo "replace script repo with namespace repo" + find . -type f -exec sed -i "s|dataset_path: $repo_name|dataset_path: $repo|g" {} + + fi + # search full id to cover mirror repo id + mapfile -t yaml_files < <(grep -Rl -E "(dataset_path: ${repo}($|\s))" $task_dir) + file_count=${#yaml_files[@]} + if [ "$file_count" -eq 0 ]; then + # search short id to cover csghub repo id + mapfile -t yaml_files < <(grep -Rl -E "(dataset_path: .*/${repo_name}($|\s))|(dataset_path: ${repo_name}($|\s))" $task_dir) + fi + file_count=${#yaml_files[@]} + if [ "$file_count" -eq 0 ]; then + echo "no yaml file found for repo $repo" + continue + fi + # check yaml_files size + common_path="${yaml_files[0]}" + if [ "$file_count" -gt 1 ]; then + for path in "${yaml_files[@]}"; do + while [[ "$path" != "${common_path}"* ]]; do + common_path="${common_path%/*}" + done + done + if [ "x$common_path" == "x$task_dir" ]; then + echo "no common path found for repo $repo, will pick one of the yaml_files" + matched_path=$(search_path_with_most_term "$repo_name" "${yaml_files[@]}") + common_path=$(dirname "$matched_path") + fi + else + common_path=$(dirname "$common_path") + fi + echo "common path found for repo $repo: $common_path" + repo_task=`python /etc/csghub/get_task.py $common_path` + if [ ! -z "$repo_task" ]; then + tasks="$tasks,$repo_task" + fi +done +tasks=$(echo "$tasks" | sed 's/^,//; s/,$//') +tasks=$(echo "$tasks" | tr -d ' ' | tr ',' ',') +echo "will start tasks: $tasks" + + +accelerate launch -m lm_eval \ + --model hf \ + --model_args pretrained=${MODEL_ID},dtype=auto,trust_remote_code=True \ + --tasks "$tasks" \ + --batch_size auto \ + --output_path /workspace/output/ + +if [ $? -eq 0 ]; then + echo "Evaluation completed successfully." +else + echo "Evaluation failed." + exit 1 +fi + +# upload result to mino server +json_file=`ls -dt /workspace/output/*/*.json |head -n 1` +model_name=`basename $MODEL_ID` +python /etc/csghub/upload_files.py summary --file $json_file --model $model_name +upload_json_file=`ls -d /workspace/output/*${model_name}*/*upload.json` +upload_xlsx_file=`ls -d /workspace/output/*${model_name}*/*upload.xlsx` +python /etc/csghub/upload_files.py upload "$upload_json_file,$upload_xlsx_file" +echo "finish evaluation for $MODEL_ID" \ No newline at end of file diff --git a/docker/evaluation/lm-evaluation-harness/upload_files.py b/docker/evaluation/lm-evaluation-harness/upload_files.py new file mode 100644 index 00000000..668a1cd5 --- /dev/null +++ b/docker/evaluation/lm-evaluation-harness/upload_files.py @@ -0,0 +1,222 @@ +import os +import argparse +from datetime import datetime +from minio import Minio +from minio.error import S3Error +from pathlib import Path +import pandas as pd +import json +import oss2 +import csv + +access_key_id = os.environ['S3_ACCESS_ID'] +access_key_secret = os.environ['S3_ACCESS_SECRET'] +bucket_name = os.environ['S3_BUCKET'] +endpoint = os.environ['S3_ENDPOINT'] +s3_ssl_enabled = json.loads(os.environ['S3_SSL_ENABLED']) +if endpoint.find("aliyuncs.com") == -1: + client = Minio(endpoint, access_key=access_key_id, secret_key=access_key_secret, secure=s3_ssl_enabled) +else: + auth = oss2.Auth(access_key_id, access_key_secret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + + +def generate_file_name(name): + # Get the current date and time + now = datetime.now() + + # Format the string as YYYYMMDD_HHMMSS + formatted_uuid = now.strftime("%Y%m%d_%H%M%S") + + return f"{name}_{formatted_uuid}" + + +def upload_to_minio(object_name, location_file): + # Make the bucket if it doesn't exist. + found = client.bucket_exists(bucket_name) + if not found: + client.make_bucket(bucket_name) + print("Created bucket", bucket_name) + else: + print("Bucket", bucket_name, "already exists") + + # Upload the file, renaming it in the process + client.fput_object( + bucket_name, object_name, location_file, + ) + + +def upload_to_ali(object_name, location_file): + # Upload the file, renaming it in the process + bucket.put_object_from_file(object_name, location_file) + + +def upload(files): + output = [] + # get schema based on s3_ssl_enabled + schema = "https" if s3_ssl_enabled else "http" + fileName = generate_file_name("result") + for file in files.split(','): + suffix = Path(file).suffix + object_name = f"evaluation/{fileName}{suffix}" + # check if the endpoint is aliyun oss + if endpoint.find("aliyuncs.com") != -1: + upload_to_ali(object_name, file) + file_url = f"https://{bucket_name}.{endpoint}/{object_name}" + else: + upload_to_minio(object_name, file) + file_url = f"{schema}://{endpoint}/{bucket_name}/{object_name}" + output.append(file_url) + try: + with open('/tmp/output.txt', 'w') as file: + file.write(",".join(output)) + print("Output written to /tmp/output.txt") + print(f'Successfully uploaded to {file_url}') + except Exception as e: + print(f"Error writing to file: {e}") + + +column = [ + { + "title": { + "zh-CN": "数据集", + "en-US": "Dataset" + }, + "width": 220, + "key": "dataset", + "fixed": "left" + }, + { + "title": { + "zh-CN": "指标", + "en-US": "Metric" + }, + "width": 130, + "key": "metric", + "fixed": "left" + }, + { + "title": { + "zh-CN": "模式", + "en-US": "Mode" + }, + "width": 100, + "key": "mode", + "fixed": "left" + } +] + + +def get_metric_value(resultObj, key, metric): + for item, value in resultObj[key].items(): + if metric in item: + return round(value*100, 2) + + +def get_metric_items(jsonObj, item, model_name): + metric_items = [] + metrics = jsonObj['higher_is_better'][item].keys() + for metric in metrics: + value = get_metric_value(jsonObj['results'], item, metric) + item_new = { + "dataset": item, + "version": jsonObj['versions'][item], + "metric": metric, + model_name: value, + } + metric_items.append(item_new) + return metric_items + + +def is_a_sub_group(jsonObj, key): + for k, v in jsonObj["group_subtasks"].items(): + if key in v: + return True + return False + + +def json_to_summary(jsonPath, model_name): + with open(jsonPath, 'r', encoding='utf-8') as f: + jsonObj = json.load(f) + summary_data = [] + xlsx_json = {} + # generate summary data + for k in jsonObj["group_subtasks"].keys(): + m_items = get_metric_items(jsonObj, k, model_name) + summary_data.extend(m_items) + column.append( + { + "title": { + "zh-CN": model_name, + "en-US": model_name + }, + "width": 200, + "key": model_name, + "customizeRender": { + "sorter": "descend" + } + } + ) + summary = { + "column": column, + "data": summary_data + } + final_json = {"summary": summary} + xlsx_json['summary'] = summary_data + # generate detail data + for key, value in jsonObj["group_subtasks"].items(): + sub_data = [] + if len(value) == 0: + m_items = get_metric_items(jsonObj, key, model_name) + sub_data.extend(m_items) + xlsx_json[key] = sub_data + final_json[key] = {"column": column, "data": sub_data} + else: + sub_g = is_a_sub_group(jsonObj, key) + if not sub_g: + # loop root group + for sub_key in value: + m_items = get_metric_items(jsonObj, sub_key, model_name) + sub_data.extend(m_items) + if sub_key not in jsonObj["group_subtasks"]: + continue + # level 2 group + value2 = jsonObj["group_subtasks"][sub_key] + if len(value2) == 0: + m_items = get_metric_items(jsonObj, key, model_name) + sub_data.extend(m_items) + else: + for sub_sub_key in value2: + m_items = get_metric_items(jsonObj, sub_sub_key, model_name) + sub_data.extend(m_items) + xlsx_json[key] = sub_data + final_json[key] = {"column": column, "data": sub_data} + + json_file_path = os.path.splitext(jsonPath)[0] + '_upload.json' + with open(json_file_path, 'w', encoding='utf-8') as f: + json.dump(final_json, f, ensure_ascii=False, indent=4) + + xlsx_file = os.path.splitext(jsonPath)[0] + '_upload.xlsx' + with pd.ExcelWriter(xlsx_file) as writer: + for sheet_name, records in xlsx_json.items(): + df = pd.DataFrame(records) + df.to_excel(writer, sheet_name=sheet_name, index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get upload files.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_a = subparsers.add_parser('upload', help='upload files') + parser_a.add_argument('files', type=str, help='Name to greet') + + parser_c = subparsers.add_parser('summary', help='Convert json to json summary') + parser_c.add_argument('--file', type=str, help='Convert json to json summary') + parser_c.add_argument('--model', type=str, help='model name') + + args = parser.parse_args() + + if args.command == 'upload': + upload(args.files) + elif args.command == 'summary': + json_to_summary(args.file, args.model) diff --git a/docker/evaluation/opencompass/get_answer_mode.py b/docker/evaluation/opencompass/get_answer_mode.py new file mode 100644 index 00000000..b63b7ace --- /dev/null +++ b/docker/evaluation/opencompass/get_answer_mode.py @@ -0,0 +1,30 @@ +import argparse +from opencompass.utils.datasets_info import DATASETS_MAPPING + + +def get_dataset_key_by_ms_id(ms_id): + for key, value in DATASETS_MAPPING.items(): + if value['ms_id'] == ms_id: + return key + return None + + +def main(ms_id): + names = ms_id.split("/") + # Extract the last part + name = names[-1] + # Generate the new string + opencompass_id = f"opencompass/{name}" + dataset_key = get_dataset_key_by_ms_id(opencompass_id) + if dataset_key: + print(f"{dataset_key}") + else: + return + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get dataset key by ms_id.') + parser.add_argument('ms_id', type=str, help='The ms_id to search for') + args = parser.parse_args() + + main(args.ms_id) diff --git a/docker/evaluation/opencompass/start.sh b/docker/evaluation/opencompass/start.sh new file mode 100644 index 00000000..8ee5ef04 --- /dev/null +++ b/docker/evaluation/opencompass/start.sh @@ -0,0 +1,91 @@ +#!/bin/bash +mkdir -p /workspace/data +ANSWER_MODE=${ANSWER_MODE:-"gen"} +# download model +csghub-cli download $MODEL_ID -k $ACCESS_TOKEN -e $HF_ENDPOINT -cd /workspace/ +insert_string=$(cat << 'EOF' +"chat_template":"{%- if messages[0]['role'] == 'system' -%}{%- set system_message = messages[0]['content'] -%}{%- set messages = messages[1:] -%}{%- else -%}{% set system_message = '' -%}{%- endif -%}{{ bos_token + system_message }}{%- for message in messages -%}{%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{%- endif -%}{%- if message['role'] == 'user' -%}{{ 'USER: ' + message['content'] + '\\n' }}{%- elif message['role'] == 'assistant' -%}{{ 'ASSISTANT: ' + message['content'] + eos_token + '\\n' }}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{ 'ASSISTANT:' }}{% endif %}", +EOF +) +repo_tokenizer_config="/workspace/$MODEL_ID/tokenizer_config.json" +# check if repo_tokenizer_config exists +if [ ! -f "$repo_tokenizer_config" ]; then + echo "the model is invalid." + exit 1 +fi + +# fix some model does not contain chat_template +if ! grep -q "chat_template" "$repo_tokenizer_config"; then + filename="/tmp/tokenizer_config.json" + cp "/workspace/$MODEL_ID/tokenizer_config.json" $filename + awk -v ins="$insert_string" '/tokenizer_class/ {print; print ins; next}1' "$filename" > tmpfile && mv -f tmpfile $repo_tokenizer_config +fi +# download datasets +IFS=',' read -r -a dataset_repos <<< "$DATASET_IDS" +# Loop through the array and print each value +dataset_tasks="" +dataset_tasks_ori="" +for repo in "${dataset_repos[@]}"; do + # check $dataset existing + if [ ! -d "/workspace/data/$repo" ]; then + echo "Start downloading dataset $repo..." + csghub-cli download $repo -t dataset -k $ACCESS_TOKEN -e $HF_ENDPOINT -cd /tmp/ + # mv "$dataset" to "/workspace/data/" + mv -f "/tmp/$repo" "/workspace/data/" + fi + # get answer mode + task_path=`python -W ignore /etc/csghub/get_answer_mode.py $repo` + if [ -z "$task_path" ]; then + echo "task_path is empty for dataset $repo" + exit 1 + fi + datasets_conf_dir="/usr/local/lib/python3.10/dist-packages/opencompass/configs/datasets/" + dataset_conf_file=`find $datasets_conf_dir -type f -name "*.py" -exec grep -rl "'$task_path'" {} + | head -n 1` + if [ -z "$dataset_conf_file" ]; then + echo "Cannot find dataset config location for $task_path" + exit 1 + fi + #get dataset_location dir + dataset_conf_dir=`dirname $dataset_conf_file` + task_conf_file=`find $dataset_conf_dir -type f -name "*$ANSWER_MODE.py" | head -n 1` + if [ -n "$task_conf_file" ]; then + task=`basename $task_conf_file | cut -d'.' -f1` + dataset_tasks="$dataset_tasks $task" + ori_name=`basename $repo` + dataset_tasks_ori="$dataset_tasks_ori $ori_name" + continue + fi +done +# start evaluation +if [ -z "$dataset_tasks" ]; then + echo "dataset_tasks is empty for dataset $DATASET_IDS" + exit 1 +fi +if [ -z "$GPU_NUM" ]; then + GPU_NUM=1 +fi +#LimitedMaxToken is gpu_num multiplied by 4096 +LimitedMaxToken=$(($GPU_NUM * 4096)) +# avoid GPU OOM +repo_config="/workspace/$MODEL_ID/config.json" +sed -i "s/\"max_position_embeddings\": [0-9]*/\"max_position_embeddings\": $LimitedMaxToken/" $repo_config + +opencompass --datasets $dataset_tasks --work-dir /workspace/output --hf-type chat --hf-path /workspace/$MODEL_ID -a vllm --max-out-len 100 --max-seq-len $LimitedMaxToken --batch-size 8 --hf-num-gpus $GPU_NUM --max-num-workers $GPU_NUM + +if [ $? -eq 0 ]; then + echo "Evaluation completed successfully." +else + echo "Evaluation failed." + exit 1 +fi + +# upload result to mino server +output_dir=`ls -dt /workspace/output/* |head -n 1` +csv_file=`ls -d $output_dir/summary/*.csv` +python /etc/csghub/upload_files.py convert "$csv_file" +json_file=`ls -d $output_dir/summary/*.json` +python /etc/csghub/upload_files.py summary --file $json_file --tasks $dataset_tasks_ori +upload_json_file=`ls -d $output_dir/summary/*upload.json` +upload_xlsx_file=`ls -d $output_dir/summary/*upload.xlsx` +python /etc/csghub/upload_files.py upload "$upload_json_file,$upload_xlsx_file" +echo "finish evaluation for $MODEL_ID" diff --git a/docker/evaluation/opencompass/upload_files.py b/docker/evaluation/opencompass/upload_files.py new file mode 100644 index 00000000..9c9bff30 --- /dev/null +++ b/docker/evaluation/opencompass/upload_files.py @@ -0,0 +1,201 @@ +import os +import argparse +from datetime import datetime +from minio import Minio +from minio.error import S3Error +from pathlib import Path +import pandas as pd +import json +import oss2 +import csv + +access_key_id = os.environ['S3_ACCESS_ID'] +access_key_secret = os.environ['S3_ACCESS_SECRET'] +bucket_name = os.environ['S3_BUCKET'] +endpoint = os.environ['S3_ENDPOINT'] +s3_ssl_enabled = json.loads(os.environ['S3_SSL_ENABLED']) +if endpoint.find("aliyuncs.com") == -1: + client = Minio(endpoint, access_key=access_key_id, secret_key=access_key_secret, secure=s3_ssl_enabled) +else: + auth = oss2.Auth(access_key_id, access_key_secret) + bucket = oss2.Bucket(auth, endpoint, bucket_name) + + +def generate_file_name(name): + # Get the current date and time + now = datetime.now() + + # Format the string as YYYYMMDD_HHMMSS + formatted_uuid = now.strftime("%Y%m%d_%H%M%S") + + return f"{name}_{formatted_uuid}" + + +def upload_to_minio(object_name, location_file): + # Make the bucket if it doesn't exist. + found = client.bucket_exists(bucket_name) + if not found: + client.make_bucket(bucket_name) + print("Created bucket", bucket_name) + else: + print("Bucket", bucket_name, "already exists") + + # Upload the file, renaming it in the process + client.fput_object( + bucket_name, object_name, location_file, + ) + + +def upload_to_ali(object_name, location_file): + # Upload the file, renaming it in the process + bucket.put_object_from_file(object_name, location_file) + + +def upload(files): + output = [] + # get schema based on s3_ssl_enabled + schema = "https" if s3_ssl_enabled else "http" + fileName = generate_file_name("result") + for file in files.split(','): + suffix = Path(file).suffix + object_name = f"evaluation/{fileName}{suffix}" + # check if the endpoint is aliyun oss + if endpoint.find("aliyuncs.com") != -1: + upload_to_ali(object_name, file) + file_url = f"https://{bucket_name}.{endpoint}/{object_name}" + else: + upload_to_minio(object_name, file) + file_url = f"{schema}://{endpoint}/{bucket_name}/{object_name}" + output.append(file_url) + try: + with open('/tmp/output.txt', 'w') as file: + file.write(",".join(output)) + print("Output written to /tmp/output.txt") + print(f'Successfully uploaded to {file_url}') + except Exception as e: + print(f"Error writing to file: {e}") + + +def csv_to_json(csv_file_path): + # Read the CSV file + with open(csv_file_path, mode='r', newline='', encoding='utf-8') as csv_file: + csv_reader = csv.DictReader(csv_file) # Using DictReader to read as dictionaries + data = list(csv_reader) # Convert to a list of dictionaries + + # Write the JSON output + json_file_path = os.path.splitext(csv_file_path)[0] + '.json' + with open(json_file_path, mode='w', encoding='utf-8') as json_file: + json.dump(data, json_file, indent=4) # Pretty print the JSON output + + print(f'Successfully converted {csv_file_path} to {json_file_path}') + + +column = [ + { + "title": { + "zh-CN": "数据集", + "en-US": "Dataset" + }, + "width": 220, + "key": "dataset", + "fixed": "left" + }, + { + "title": { + "zh-CN": "指标", + "en-US": "Metric" + }, + "width": 130, + "key": "metric", + "fixed": "left" + }, + { + "title": { + "zh-CN": "模式", + "en-US": "Mode" + }, + "width": 100, + "key": "mode", + "fixed": "left" + } +] + + +def json_to_summary(jsonPath, tasks): + with open(jsonPath, 'r', encoding='utf-8') as f: + jsonObj = json.load(f) + keywords = tasks + summary_data = [] + model_name = "" + xlsx_json = {} + # generate summary data + for item in jsonObj: + item_new = item.copy() + if item_new["dataset"] in keywords: + keys = list(item_new.keys()) + model_name = keys[-1] + item_new['id'] = len(summary_data) + 1 + summary_data.append(item_new) + column.append( + { + "title": { + "zh-CN": model_name, + "en-US": model_name + }, + "width": 200, + "key": model_name, + "customizeRender": { + "sorter": "descend" + } + } + ) + summary = { + "column": column, + "data": summary_data + } + final_json = {"summary": summary} + xlsx_json['summary'] = summary_data + # generate detail data + for item in keywords: + sub_data = [] + for sub_item in jsonObj: + item_new = sub_item.copy() + if item in item_new["dataset"]: + item_new['id'] = len(sub_data) + 1 + sub_data.append(item_new) + xlsx_json[item] = sub_data + final_json[item] = {"column": column, "data": sub_data} + + json_file_path = os.path.splitext(jsonPath)[0] + '_upload.json' + with open(json_file_path, 'w', encoding='utf-8') as f: + json.dump(final_json, f, ensure_ascii=False, indent=4) + + xlsx_file = os.path.splitext(jsonPath)[0] + '_upload.xlsx' + with pd.ExcelWriter(xlsx_file) as writer: + for sheet_name, records in xlsx_json.items(): + df = pd.DataFrame(records) + df.to_excel(writer, sheet_name=sheet_name, index=False) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Get upload files.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_a = subparsers.add_parser('upload', help='upload files') + parser_a.add_argument('files', type=str, help='Name to greet') + + parser_b = subparsers.add_parser('convert', help='Convert csv to json') + parser_b.add_argument('file', type=str, help='Convert csv to json') + + parser_c = subparsers.add_parser('summary', help='Convert json to json summary') + parser_c.add_argument('--file', type=str, help='Convert json to json summary') + parser_c.add_argument('--tasks', nargs='+', type=str, help='task list') + + args = parser.parse_args() + + if args.command == 'upload': + upload(args.files) + elif args.command == 'convert': + csv_to_json(args.file) + elif args.command == 'summary': + json_to_summary(args.file, args.tasks) diff --git a/docker/inference/Dockerfile.tgi b/docker/inference/Dockerfile.tgi index fe99daf2..562a4713 100644 --- a/docker/inference/Dockerfile.tgi +++ b/docker/inference/Dockerfile.tgi @@ -1,9 +1,7 @@ FROM ghcr.io/huggingface/text-generation-inference:2.4.0 RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple -RUN pip install --no-cache-dir csghub-sdk==0.4.3 supervisor -RUN apt-get update && apt-get install -y supervisor -RUN mkdir -p /var/log/supervisord -COPY ./supervisord.conf /etc/supervisor/conf.d/supervisord.conf +RUN pip install --no-cache-dir csghub-sdk==0.4.5 +RUN apt-get update && apt-get install -y dumb-init && apt-get clean && rm -rf /var/lib/apt/lists/* COPY ./tgi/ /etc/csghub/ RUN chmod +x /etc/csghub/*.sh @@ -12,5 +10,5 @@ ENV HUGGINGFACE_HUB_CACHE=/data \ HF_HUB_ENABLE_HF_TRANSFER=0 ENV PORT=8000 EXPOSE 8000 - -ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +ENTRYPOINT [ "/usr/bin/dumb-init", "--" ] +CMD ["/etc/csghub/serve.sh"] diff --git a/docker/inference/supervisord.conf b/docker/inference/supervisord.conf index ee671815..06cf5201 100644 --- a/docker/inference/supervisord.conf +++ b/docker/inference/supervisord.conf @@ -7,6 +7,7 @@ pidfile=/var/run/supervisord.pid [program:inference] +autorestart=false stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr diff --git a/docs/docs.go b/docs/docs.go index 45b5624f..3b75dac4 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -2182,6 +2182,164 @@ const docTemplate = `{ } } }, + "/evaluations": { + "post": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "run model evaluation", + "parameters": [ + { + "type": "string", + "description": "namespace", + "name": "namespace", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "body setting of evaluation", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/types.EvaluationReq" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, + "/evaluations/{id}": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "get model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.EvaluationRes" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "delete model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/events": { "post": { "consumes": [ @@ -9882,6 +10040,24 @@ const docTemplate = `{ "Tag" ], "summary": "Get all tags", + "parameters": [ + { + "type": "string", + "description": "category name", + "name": "category", + "in": "query" + }, + { + "enum": [ + "model", + "dataset" + ], + "type": "string", + "description": "scope name", + "name": "scope", + "in": "query" + } + ], "responses": { "200": { "description": "tags", @@ -10646,6 +10822,69 @@ const docTemplate = `{ } } }, + "/user/{username}/evaluations": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "description": "get user evaluations", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "User" + ], + "summary": "Get user evaluations", + "parameters": [ + { + "type": "string", + "description": "username", + "name": "username", + "in": "path", + "required": true + }, + { + "type": "integer", + "default": 20, + "description": "per", + "name": "per", + "in": "query" + }, + { + "type": "integer", + "default": 1, + "description": "per page", + "name": "page", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.Response" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/user/{username}/finetune/instances": { "get": { "security": [ @@ -14277,7 +14516,8 @@ const docTemplate = `{ "enum": [ 0, 1, - 2 + 2, + 4 ], "type": "integer", "default": 1, @@ -15123,6 +15363,9 @@ const docTemplate = `{ "id": { "type": "integer" }, + "last_updated_at": { + "type": "string" + }, "repository": { "$ref": "#/definitions/database.Repository" }, @@ -16901,6 +17144,104 @@ const docTemplate = `{ } } }, + "types.EvaluationReq": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "type": "string" + } + }, + "model_id": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "runtime_framework_id": { + "description": "ArgoWorkFlow framework", + "type": "integer" + }, + "share_mode": { + "type": "boolean" + }, + "task_desc": { + "type": "string" + }, + "task_name": { + "type": "string" + } + } + }, + "types.EvaluationRes": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTags" + } + }, + "download_url": { + "type": "string" + }, + "end_time": { + "type": "string" + }, + "failures_url": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "image": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "repo_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_type": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "result_url": { + "type": "string" + }, + "start_time": { + "type": "string" + }, + "status": { + "type": "string" + }, + "submit_time": { + "type": "string" + }, + "task_desc": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "task_type": { + "$ref": "#/definitions/types.TaskType" + }, + "username": { + "type": "string" + } + } + }, "types.Event": { "type": "object", "properties": { @@ -17161,6 +17502,9 @@ const docTemplate = `{ "downloads": { "type": "integer" }, + "enable_evaluation": { + "type": "boolean" + }, "enable_finetune": { "type": "boolean" }, @@ -17197,12 +17541,18 @@ const docTemplate = `{ "readme": { "type": "string" }, + "recom_op_weight": { + "type": "integer" + }, "repository": { "$ref": "#/definitions/types.Repository" }, "repository_id": { "type": "integer" }, + "sensitive_check_status": { + "type": "string" + }, "source": { "$ref": "#/definitions/types.RepositorySource" }, @@ -17569,6 +17919,20 @@ const docTemplate = `{ } } }, + "types.RepoTags": { + "type": "object", + "properties": { + "repo_id": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTag" + } + } + } + }, "types.Repository": { "type": "object", "properties": { @@ -17959,6 +18323,21 @@ const docTemplate = `{ } } }, + "types.TaskType": { + "type": "string", + "enum": [ + "evaluation", + "training", + "comparison", + "leaderboard" + ], + "x-enum-varnames": [ + "TaskTypeEvaluation", + "TaskTypeTraining", + "TaskTypeComparison", + "TaskTypeLeaderBoard" + ] + }, "types.UpdateCodeReq": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 1922d407..c5dfdef4 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -2171,6 +2171,164 @@ } } }, + "/evaluations": { + "post": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "run model evaluation", + "parameters": [ + { + "type": "string", + "description": "namespace", + "name": "namespace", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "name", + "name": "name", + "in": "path", + "required": true + }, + { + "description": "body setting of evaluation", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/types.EvaluationReq" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, + "/evaluations/{id}": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "get model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.EvaluationRes" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + }, + "delete": { + "security": [ + { + "ApiKey": [] + } + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Evaluation" + ], + "summary": "delete model evaluation", + "parameters": [ + { + "type": "string", + "description": "id", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "string" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/events": { "post": { "consumes": [ @@ -9871,6 +10029,24 @@ "Tag" ], "summary": "Get all tags", + "parameters": [ + { + "type": "string", + "description": "category name", + "name": "category", + "in": "query" + }, + { + "enum": [ + "model", + "dataset" + ], + "type": "string", + "description": "scope name", + "name": "scope", + "in": "query" + } + ], "responses": { "200": { "description": "tags", @@ -10635,6 +10811,69 @@ } } }, + "/user/{username}/evaluations": { + "get": { + "security": [ + { + "ApiKey": [] + } + ], + "description": "get user evaluations", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "User" + ], + "summary": "Get user evaluations", + "parameters": [ + { + "type": "string", + "description": "username", + "name": "username", + "in": "path", + "required": true + }, + { + "type": "integer", + "default": 20, + "description": "per", + "name": "per", + "in": "query" + }, + { + "type": "integer", + "default": 1, + "description": "per page", + "name": "page", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/types.Response" + } + }, + "400": { + "description": "Bad request", + "schema": { + "$ref": "#/definitions/types.APIBadRequest" + } + }, + "500": { + "description": "Internal server error", + "schema": { + "$ref": "#/definitions/types.APIInternalServerError" + } + } + } + } + }, "/user/{username}/finetune/instances": { "get": { "security": [ @@ -14266,7 +14505,8 @@ "enum": [ 0, 1, - 2 + 2, + 4 ], "type": "integer", "default": 1, @@ -15112,6 +15352,9 @@ "id": { "type": "integer" }, + "last_updated_at": { + "type": "string" + }, "repository": { "$ref": "#/definitions/database.Repository" }, @@ -16890,6 +17133,104 @@ } } }, + "types.EvaluationReq": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "type": "string" + } + }, + "model_id": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "runtime_framework_id": { + "description": "ArgoWorkFlow framework", + "type": "integer" + }, + "share_mode": { + "type": "boolean" + }, + "task_desc": { + "type": "string" + }, + "task_name": { + "type": "string" + } + } + }, + "types.EvaluationRes": { + "type": "object", + "properties": { + "datasets": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTags" + } + }, + "download_url": { + "type": "string" + }, + "end_time": { + "type": "string" + }, + "failures_url": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "image": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "repo_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "repo_type": { + "type": "string" + }, + "resource_id": { + "type": "integer" + }, + "result_url": { + "type": "string" + }, + "start_time": { + "type": "string" + }, + "status": { + "type": "string" + }, + "submit_time": { + "type": "string" + }, + "task_desc": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "task_type": { + "$ref": "#/definitions/types.TaskType" + }, + "username": { + "type": "string" + } + } + }, "types.Event": { "type": "object", "properties": { @@ -17150,6 +17491,9 @@ "downloads": { "type": "integer" }, + "enable_evaluation": { + "type": "boolean" + }, "enable_finetune": { "type": "boolean" }, @@ -17186,12 +17530,18 @@ "readme": { "type": "string" }, + "recom_op_weight": { + "type": "integer" + }, "repository": { "$ref": "#/definitions/types.Repository" }, "repository_id": { "type": "integer" }, + "sensitive_check_status": { + "type": "string" + }, "source": { "$ref": "#/definitions/types.RepositorySource" }, @@ -17558,6 +17908,20 @@ } } }, + "types.RepoTags": { + "type": "object", + "properties": { + "repo_id": { + "type": "string" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/types.RepoTag" + } + } + } + }, "types.Repository": { "type": "object", "properties": { @@ -17948,6 +18312,21 @@ } } }, + "types.TaskType": { + "type": "string", + "enum": [ + "evaluation", + "training", + "comparison", + "leaderboard" + ], + "x-enum-varnames": [ + "TaskTypeEvaluation", + "TaskTypeTraining", + "TaskTypeComparison", + "TaskTypeLeaderBoard" + ] + }, "types.UpdateCodeReq": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 7386d230..c8f611fc 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -183,6 +183,8 @@ definitions: type: string id: type: integer + last_updated_at: + type: string repository: $ref: '#/definitions/database.Repository' repository_id: @@ -1383,6 +1385,71 @@ definitions: example: false type: boolean type: object + types.EvaluationReq: + properties: + datasets: + items: + type: string + type: array + model_id: + type: string + resource_id: + type: integer + runtime_framework_id: + description: ArgoWorkFlow framework + type: integer + share_mode: + type: boolean + task_desc: + type: string + task_name: + type: string + type: object + types.EvaluationRes: + properties: + datasets: + items: + $ref: '#/definitions/types.RepoTags' + type: array + download_url: + type: string + end_time: + type: string + failures_url: + type: string + id: + type: integer + image: + type: string + reason: + type: string + repo_ids: + items: + type: string + type: array + repo_type: + type: string + resource_id: + type: integer + result_url: + type: string + start_time: + type: string + status: + type: string + submit_time: + type: string + task_desc: + type: string + task_id: + type: string + task_name: + type: string + task_type: + $ref: '#/definitions/types.TaskType' + username: + type: string + type: object types.Event: properties: c_id: @@ -1562,6 +1629,8 @@ definitions: type: string downloads: type: integer + enable_evaluation: + type: boolean enable_finetune: type: boolean enable_inference: @@ -1586,10 +1655,14 @@ definitions: type: boolean readme: type: string + recom_op_weight: + type: integer repository: $ref: '#/definitions/types.Repository' repository_id: type: integer + sensitive_check_status: + type: string source: $ref: '#/definitions/types.RepositorySource' status: @@ -1831,6 +1904,15 @@ definitions: updated_at: type: string type: object + types.RepoTags: + properties: + repo_id: + type: string + tags: + items: + $ref: '#/definitions/types.RepoTag' + type: array + type: object types.Repository: properties: http_clone_url: @@ -2103,6 +2185,18 @@ definitions: msg: type: string type: object + types.TaskType: + enum: + - evaluation + - training + - comparison + - leaderboard + type: string + x-enum-varnames: + - TaskTypeEvaluation + - TaskTypeTraining + - TaskTypeComparison + - TaskTypeLeaderBoard types.UpdateCodeReq: properties: description: @@ -3726,6 +3820,7 @@ paths: - 0 - 1 - 2 + - 4 in: query name: deploy_type type: integer @@ -5448,6 +5543,106 @@ paths: summary: Create a new discussion comment tags: - Discussion + /evaluations: + post: + consumes: + - application/json + parameters: + - description: namespace + in: path + name: namespace + required: true + type: string + - description: name + in: path + name: name + required: true + type: string + - description: body setting of evaluation + in: body + name: body + required: true + schema: + $ref: '#/definitions/types.EvaluationReq' + produces: + - application/json + responses: + "200": + description: OK + schema: + type: string + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: run model evaluation + tags: + - Evaluation + /evaluations/{id}: + delete: + consumes: + - application/json + parameters: + - description: id + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + type: string + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: delete model evaluation + tags: + - Evaluation + get: + consumes: + - application/json + parameters: + - description: id + in: path + name: id + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/types.EvaluationRes' + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: get model evaluation + tags: + - Evaluation /events: post: consumes: @@ -10287,6 +10482,18 @@ paths: consumes: - application/json description: get all tags + parameters: + - description: category name + in: query + name: category + type: string + - description: scope name + enum: + - model + - dataset + in: query + name: scope + type: string produces: - application/json responses: @@ -10770,6 +10977,47 @@ paths: summary: Get user datasets tags: - User + /user/{username}/evaluations: + get: + consumes: + - application/json + description: get user evaluations + parameters: + - description: username + in: path + name: username + required: true + type: string + - default: 20 + description: per + in: query + name: per + type: integer + - default: 1 + description: per page + in: query + name: page + type: integer + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/types.Response' + "400": + description: Bad request + schema: + $ref: '#/definitions/types.APIBadRequest' + "500": + description: Internal server error + schema: + $ref: '#/definitions/types.APIInternalServerError' + security: + - ApiKey: [] + summary: Get user evaluations + tags: + - User /user/{username}/finetune/instances: get: consumes: diff --git a/go.mod b/go.mod index 984d225f..b19433b5 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/d5/tengo/v2 v2.17.0 github.com/gin-contrib/cors v1.7.0 github.com/gin-contrib/sessions v0.0.5 - github.com/gin-gonic/gin v1.9.1 + github.com/gin-gonic/gin v1.10.0 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f github.com/google/wire v0.6.0 @@ -27,7 +27,7 @@ require ( github.com/naoina/toml v0.1.1 github.com/redis/go-redis/v9 v9.3.0 github.com/sethvargo/go-envconfig v1.1.0 - github.com/spf13/cast v1.1.0 + github.com/spf13/cast v1.5.1 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 github.com/swaggo/files v1.0.1 @@ -46,9 +46,9 @@ require ( google.golang.org/grpc v1.66.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.31.0 - k8s.io/apimachinery v0.31.0 - k8s.io/client-go v0.31.0 + k8s.io/api v0.31.3 + k8s.io/apimachinery v0.31.3 + k8s.io/client-go v0.31.3 knative.dev/serving v0.40.1 ) @@ -65,8 +65,11 @@ require ( github.com/aws/aws-sdk-go v1.50.36 // indirect github.com/beevik/ntp v1.3.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect github.com/containerd/containerd v1.7.18 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect @@ -87,7 +90,9 @@ require ( github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hashicorp/yamux v0.1.2-0.20220728231024-8f49b6f63f18 // indirect github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20210210170715-a8dfcb80d3a7 // indirect github.com/lightstep/lightstep-tracer-go v0.25.0 // indirect @@ -102,6 +107,7 @@ require ( github.com/naoina/go-stringutil v0.1.0 // indirect github.com/nats-io/nkeys v0.4.7 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/nexus-rpc/sdk-go v0.0.11 // indirect github.com/oklog/ulid/v2 v2.0.2 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect @@ -142,6 +148,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20240827150818-7e3bb234dfed // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.32.0 // indirect + modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect ) require ( @@ -154,9 +161,10 @@ require ( github.com/alibabacloud-go/tea-utils/v2 v2.0.4 // indirect github.com/alibabacloud-go/tea-xml v1.1.3 // indirect github.com/aliyun/credentials-go v1.3.2 // indirect + github.com/argoproj/argo-workflows/v3 v3.5.12 github.com/blendle/zapdriver v1.3.1 // indirect github.com/bwmarrin/snowflake v0.3.0 - github.com/bytedance/sonic v1.11.2 // indirect + github.com/bytedance/sonic v1.11.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect @@ -178,7 +186,7 @@ require ( github.com/go-openapi/swag v0.22.7 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.19.0 + github.com/go-playground/validator/v10 v10.20.0 github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -186,7 +194,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/go-containerregistry v0.13.0 // indirect + github.com/google/go-containerregistry v0.16.1 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 github.com/gorilla/context v1.1.1 // indirect @@ -200,7 +208,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -216,7 +223,7 @@ require ( github.com/nats-io/nats.go v1.35.0 github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect - github.com/pelletier/go-toml/v2 v2.2.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.5.0 // indirect @@ -231,15 +238,14 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.26.0 - golang.org/x/mod v0.17.0 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/crypto v0.27.0 golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect @@ -248,18 +254,14 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 knative.dev/networking v0.0.0-20240116081125-ce0738abf051 // indirect knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 // indirect - lukechampine.com/uint128 v1.3.0 // indirect mellium.im/sasl v0.3.1 // indirect - modernc.org/cc/v3 v3.41.0 // indirect - modernc.org/ccgo/v3 v3.16.15 // indirect - modernc.org/libc v1.24.1 // indirect + modernc.org/libc v1.41.0 // indirect modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.7.1 // indirect - modernc.org/opt v0.1.3 // indirect - modernc.org/sqlite v1.25.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/sqlite v1.29.1 // indirect modernc.org/strutil v1.2.0 // indirect modernc.org/token v1.1.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index db898931..bcaaecbf 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ cloud.google.com/go v0.16.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/monitoring v1.18.0 h1:NfkDLQDG2UR3WYZVQE8kwSbUIEyIqJUPl+aOQdFH1T4= @@ -73,6 +74,9 @@ github.com/aliyun/credentials-go v1.1.2/go.mod h1:ozcZaMR5kLM7pwtCMEpVmQ242suV6q github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= github.com/aliyun/credentials-go v1.3.2 h1:L4WppI9rctC8PdlMgyTkF8bBsy9pyKQEzBD1bHMRl+g= github.com/aliyun/credentials-go v1.3.2/go.mod h1:tlpz4uys4Rn7Ik4/piGRrTbXy2uLKvePgQJJduE+Y5c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/argoproj/argo-workflows/v3 v3.5.12 h1:BD7fEPY+uF9Iellb/hYpsJkDlku8wAJwMVvTskUkZco= +github.com/argoproj/argo-workflows/v3 v3.5.12/go.mod h1:DecB01a8UXDCjtIh0udY8XfIMIRrWrlbob7hk/uMmg0= github.com/aws/aws-sdk-go v1.50.36 h1:PjWXHwZPuTLMR1NIb8nEjLucZBMzmf84TLoLbD8BZqk= github.com/aws/aws-sdk-go v1.50.36/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beevik/ntp v1.3.1 h1:Y/srlT8L1yQr58kyPWFPZIxRL8ttx2SRIpVYJqZIlAM= @@ -93,6 +97,10 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/casdoor/casdoor-go-sdk v0.41.0 h1:mqqoc1Jub34/OkAQqjeASRAaiy7x/5ZWtGObI08cfEk= github.com/casdoor/casdoor-go-sdk v0.41.0/go.mod h1:cMnkCQJgMYpgAlgEx8reSt1AVaDIQLcJ1zk5pzBaz+4= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -121,6 +129,10 @@ github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/tableflip v1.2.3 h1:8I+B99QnnEWPHOY3fWipwVKxS70LGgUsslG7CSfmHMw= github.com/cloudflare/tableflip v1.2.3/go.mod h1:P4gRehmV6Z2bY5ao5ml9Pd8u6kuEnlB37pUFMmv7j2E= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/containerd v1.7.18 h1:jqjZTQNfXGoEaZdW1WwPU0RqSn1Bm2Ay/KJPUuO8nao= github.com/containerd/containerd v1.7.18/go.mod h1:IYEk9/IO6wAPUz2bCMVUbsfXjzw5UNP5fLz4PsUygQ4= @@ -132,8 +144,8 @@ github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GK github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= +github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/d5/tengo/v2 v2.17.0 h1:BWUN9NoJzw48jZKiYDXDIF3QrIVZRm1uV1gTzeZ2lqM= github.com/d5/tengo/v2 v2.17.0/go.mod h1:XRGjEs5I9jYIKTxly6HCF8oiiilk5E/RYXOZ5b0DZC8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -160,8 +172,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.8.0+incompatible h1:1Av9pn2FyxPdvrWNQszj1g6D6YthSmvCfcN6SYclTJg= +github.com/evanphx/json-patch v5.8.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= @@ -171,6 +183,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.3-0.20170329110642-4da3e2cfbabc/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -181,6 +195,7 @@ github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXE github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/cors v1.7.0 h1:wZX2wuZ0o7rV2/1i7gb4Jn+gW7HBqaP91fizJkBUJOA= github.com/gin-contrib/cors v1.7.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= @@ -192,6 +207,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -229,6 +246,8 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= @@ -292,8 +311,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k= -github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo= +github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ= +github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -328,11 +347,15 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0 h1:2cz5kSrxzMYHiWOBbKj8itQm+nRykkB8aMv4ThcHYHA= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0/go.mod h1:w9Y7gY31krpLmrVU5ZPG9H7l9fZuRu5/3R3S3FMtVQ4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I= github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v0.0.0-20170914154624-68e816d1c783/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= github.com/hashicorp/yamux v0.1.2-0.20220728231024-8f49b6f63f18 h1:IVujPV6DRIu1fYF4zUHrfhkngJzmYjelXa+iSUiFZSI= github.com/hashicorp/yamux v0.1.2-0.20220728231024-8f49b6f63f18/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= @@ -371,8 +394,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= @@ -465,6 +486,8 @@ github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nexus-rpc/sdk-go v0.0.11 h1:qH3Us3spfp50t5ca775V1va2eE6z1zMQDZY4mvbw0CI= github.com/nexus-rpc/sdk-go v0.0.11/go.mod h1:TpfkM2Cw0Rlk9drGkoiSMpFqflKTiQLWUNyKJjF8mKQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -503,6 +526,8 @@ github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -532,6 +557,7 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= @@ -555,8 +581,9 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.1.0 h1:0Rhw4d6C8J9VPu6cjZLIhZ8+aAOHcDvGeKn+cq5Aq3k= github.com/spf13/cast v1.1.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v0.0.0-20170901151539-12bd96e66386/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -693,6 +720,8 @@ go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -709,6 +738,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -740,6 +771,7 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -748,6 +780,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -767,11 +800,13 @@ golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20170912212905-13449ad91cb2/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -826,6 +861,8 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -836,6 +873,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -848,6 +887,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20170424234030-8be79e1e0910/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -899,6 +940,7 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s= google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= @@ -915,6 +957,7 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= @@ -952,6 +995,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -968,10 +1012,16 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= +k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= +k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= +k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= @@ -984,36 +1034,22 @@ knative.dev/pkg v0.0.0-20240116073220-b488e7be5902 h1:H6+JJN23fhwYWCHY1339sY6uhI knative.dev/pkg v0.0.0-20240116073220-b488e7be5902/go.mod h1:NYk8mMYoLkO7CQWnNkti4YGGnvLxN6MIDbUvtgeo0C0= knative.dev/serving v0.40.1 h1:ZAAK8KwZQYUgCgVi3ay+NqKAISnJQ1OXPYvdtXWUcBc= knative.dev/serving v0.40.1/go.mod h1:Ory3XczDB8b1lH757CSdeDeouY3LHzSamX8IjmStuoU= -lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= -lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo= mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw= -modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= -modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= -modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0= -modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI= -modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= -modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= +modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6/go.mod h1:Qz0X07sNOR1jWYCrJMEnbW/X55x206Q7Vt4mz6/wHp4= +modernc.org/libc v1.41.0 h1:g9YAc6BkKlgORsUWj+JwqoB1wU3o4DE3bM3yvA3k+Gk= +modernc.org/libc v1.41.0/go.mod h1:w0eszPsiXoOnoMJgrXjglgLuDy/bt5RR4y3QzUUeodY= modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.7.1 h1:9J+2/GKTlV503mk3yv8QJ6oEpRCUrRy0ad8TXEPoV8M= -modernc.org/memory v1.7.1/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= -modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/sqlite v1.29.1 h1:19GY2qvWB4VPw0HppFlZCPAbmxFU41r+qjKZQdQ1ryA= +modernc.org/sqlite v1.29.1/go.mod h1:hG41jCYxOAOoO6BRK66AdRlmOcDzXf7qnwlwjUIOqa0= modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= -modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= -modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/servicerunner/component/service.go b/runner/component/service.go similarity index 97% rename from servicerunner/component/service.go rename to runner/component/service.go index a249f067..0b7276e0 100644 --- a/servicerunner/component/service.go +++ b/runner/component/service.go @@ -49,7 +49,7 @@ func (s *ServiceComponent) GenerateService(ctx context.Context, cluster cluster. environments := []corev1.EnvVar{} appPort := 0 hardware := request.Hardware - resReq, nodeSelector := s.GenerateResources(hardware) + resReq, nodeSelector := GenerateResources(hardware) var err error if request.Env != nil { @@ -218,7 +218,7 @@ func (s *ServiceComponent) GetServicePodsWithStatus(ctx context.Context, cluster return podInstances, nil } -func (s *ServiceComponent) GenerateResources(hardware types.HardWare) (map[corev1.ResourceName]resource.Quantity, map[string]string) { +func GenerateResources(hardware types.HardWare) (map[corev1.ResourceName]resource.Quantity, map[string]string) { nodeSelector := make(map[string]string) resReq := make(map[corev1.ResourceName]resource.Quantity) diff --git a/runner/component/workflow.go b/runner/component/workflow.go new file mode 100644 index 00000000..3b164256 --- /dev/null +++ b/runner/component/workflow.go @@ -0,0 +1,385 @@ +package component + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "path" + "strconv" + "strings" + "time" + + "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" + "github.com/argoproj/argo-workflows/v3/pkg/client/informers/externalversions" + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" + "opencsg.com/csghub-server/builder/deploy/cluster" + "opencsg.com/csghub-server/builder/event" + "opencsg.com/csghub-server/builder/store/database" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" +) + +type workFlowComponentImpl struct { + config *config.Config + wf database.ArgoWorkFlowStore + clusterPool *cluster.ClusterPool + eventPub *event.EventPublisher +} + +type WorkFlowComponent interface { + // Create workflow + CreateWorkflow(ctx context.Context, req types.ArgoWorkFlowReq) (*database.ArgoWorkflow, error) + // Update workflow + UpdateWorkflow(ctx context.Context, update *v1alpha1.Workflow) (*database.ArgoWorkflow, error) + // find workflow by user name + FindWorkFlows(ctx context.Context, username string, per, page int) ([]database.ArgoWorkflow, int, error) + // generate workflow templates + DeleteWorkflow(ctx context.Context, id int64, username string) error + GetWorkflow(ctx context.Context, id int64, username string) (*database.ArgoWorkflow, error) + DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error + FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error) + RunWorkflowsInformer(clusterPool *cluster.ClusterPool, config *config.Config) +} + +func NewWorkFlowComponent(config *config.Config, clusterPool *cluster.ClusterPool) WorkFlowComponent { + wf := database.NewArgoWorkFlowStore() + wc := &workFlowComponentImpl{ + config: config, + wf: wf, + clusterPool: clusterPool, + eventPub: &event.DefaultEventPublisher, + } + //watch workflows events + go wc.RunWorkflowsInformer(clusterPool, config) + return wc +} + +// Create workflow +func (wc *workFlowComponentImpl) CreateWorkflow(ctx context.Context, req types.ArgoWorkFlowReq) (*database.ArgoWorkflow, error) { + // create workflow in db + namespace := wc.config.Argo.Namespace + if req.ShareMode { + namespace = wc.config.Argo.QuotaNamespace + } + cluster, clusterId, err := GetCluster(ctx, wc.clusterPool, req.ClusterID) + if err != nil { + return nil, fmt.Errorf("failed to get cluster by id: %v", err) + } + argowf := &database.ArgoWorkflow{ + Username: req.Username, + UserUUID: req.UserUUID, + TaskName: req.TaskName, + TaskId: req.TaskId, + TaskType: req.TaskType, + RepoIds: req.RepoIds, + TaskDesc: req.TaskDesc, + Image: req.Image, + Datasets: req.Datasets, + ResourceId: req.ResourceId, + ResourceName: req.ResourceName, + ClusterID: clusterId, + RepoType: req.RepoType, + Namespace: namespace, + Status: v1alpha1.WorkflowPhase(v1alpha1.NodePending), + } + // create workflow in argo + awf := generateWorkflow(req, wc.config) + + _, err = cluster.ArgoClient.ArgoprojV1alpha1().Workflows(namespace).Create(ctx, awf, v1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to create workflow in argo: %v", err) + } + wf, err := wc.wf.CreateWorkFlow(ctx, *argowf) + if err != nil { + return nil, fmt.Errorf("failed to create workflow in db: %v", err) + } + return wf, nil +} + +func (wc *workFlowComponentImpl) DeleteWorkflow(ctx context.Context, id int64, username string) error { + wf, err := wc.FindWorkFlowById(ctx, id) + if err != nil { + return fmt.Errorf("failed to get workflow by id: %v", err) + } + if wf.Username != username { + return fmt.Errorf("no permission to delete workflow") + } + cluster, _, err := GetCluster(ctx, wc.clusterPool, wf.ClusterID) + if err != nil { + return fmt.Errorf("failed to get cluster by id: %v", err) + } + err = cluster.ArgoClient.ArgoprojV1alpha1().Workflows(wf.Namespace).Delete(ctx, wf.TaskId, v1.DeleteOptions{}) + if err != nil { + slog.Warn("Error deleting argo workflow", slog.Any("error", err)) + } + return wc.wf.DeleteWorkFlow(ctx, wf.ID) +} + +func (wc *workFlowComponentImpl) GetWorkflow(ctx context.Context, id int64, username string) (*database.ArgoWorkflow, error) { + wf, err := wc.FindWorkFlowById(ctx, id) + if err != nil { + return nil, fmt.Errorf("failed to get workflow by id: %v", err) + } + if wf.Username != username { + return nil, fmt.Errorf("no permission to get workflow") + } + return &wf, nil +} + +// Update workflow +func (wc *workFlowComponentImpl) UpdateWorkflow(ctx context.Context, update *v1alpha1.Workflow) (*database.ArgoWorkflow, error) { + oldwf, err := wc.wf.FindByTaskID(ctx, update.Name) + if err != nil { + return nil, err + } + + oldwf.Reason = update.Status.Message + if node, ok := update.Status.Nodes[oldwf.TaskId]; ok { + oldwf.Status = v1alpha1.WorkflowPhase(node.Phase) + if node.Phase == v1alpha1.NodeRunning { + oldwf.StartTime = time.Now() + } + if _, exists := types.WorkFlowFinished[oldwf.Status]; exists { + oldwf.EndTime = time.Now() + } + if node.Outputs != nil && node.Outputs.Parameters != nil { + for _, output := range node.Outputs.Parameters { + if output.Name == "result" && output.Value != nil { + result := strings.Split(output.Value.String(), ",") + oldwf.ResultURL = result[0] + oldwf.DownloadURL = result[1] + wc.StartAcctRequestFee(oldwf) + break + } + } + } + } + return wc.wf.UpdateWorkFlow(ctx, oldwf) +} + +// DeleteWorkflowInargo +func (wc *workFlowComponentImpl) DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error { + wf, err := wc.wf.FindByTaskID(ctx, delete.Name) + if err != nil { + return fmt.Errorf("failed to get workflow by id: %v", err) + } + // for deleted case,check if the workflow did not finish + if wf.Status == v1alpha1.WorkflowPending || wf.Status == v1alpha1.WorkflowRunning { + wf.Status = v1alpha1.WorkflowFailed + wf.Reason = "deleted by admin" + _, err = wc.wf.UpdateWorkFlow(ctx, wf) + return err + } + return nil +} + +func (wc *workFlowComponentImpl) FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error) { + return wc.wf.FindByID(ctx, id) +} + +// find workflow by user name +func (wc *workFlowComponentImpl) FindWorkFlows(ctx context.Context, username string, per, page int) ([]database.ArgoWorkflow, int, error) { + return wc.wf.FindByUsername(ctx, username, per, page) +} + +// create workflow in argo +func generateWorkflow(req types.ArgoWorkFlowReq, config *config.Config) *v1alpha1.Workflow { + templates := []v1alpha1.Template{} + for _, v := range req.Templates { + resReq, _ := GenerateResources(v.HardWare) + environments := []corev1.EnvVar{} + for key, value := range v.Env { + environments = append(environments, corev1.EnvVar{Name: key, Value: value}) + } + environments = append(environments, corev1.EnvVar{Name: "S3_ACCESS_ID", Value: config.S3.AccessKeyID}) + environments = append(environments, corev1.EnvVar{Name: "S3_ACCESS_SECRET", Value: config.S3.AccessKeySecret}) + environments = append(environments, corev1.EnvVar{Name: "S3_BUCKET", Value: config.Argo.S3PublicBucket}) + environments = append(environments, corev1.EnvVar{Name: "S3_ENDPOINT", Value: config.S3.Endpoint}) + environments = append(environments, corev1.EnvVar{Name: "S3_SSL_ENABLED", Value: strconv.FormatBool(config.S3.EnableSSL)}) + // fix no gpu request case + if v.HardWare.Gpu.ResourceName == "" || v.HardWare.Gpu.Num == "" { + environments = append(environments, corev1.EnvVar{Name: "NVIDIA_VISIBLE_DEVICES", Value: "none"}) + } + + resources := corev1.ResourceRequirements{ + Limits: resReq, + Requests: resReq, + } + + containerImg := v.Image + // add prefix if image is not full path + if !strings.Contains(containerImg, "/") { + if req.RepoType == string(types.ModelRepo) { + // choose registry + containerImg = path.Join(config.Model.DockerRegBase, v.Image) + } else if req.RepoType == string(types.SpaceRepo) { + // choose registry + containerImg = path.Join(config.Space.DockerRegBase, v.Image) + } + } + + templates = append(templates, v1alpha1.Template{ + Name: v.Name, + //NodeSelector: nodeSelector, + Container: &corev1.Container{ + Image: containerImg, + Command: v.Command, + Env: environments, + Args: v.Args, + Resources: resources, + }, + Outputs: v1alpha1.Outputs{ + Parameters: []v1alpha1.Parameter{ + { + Name: "result", + ValueFrom: &v1alpha1.ValueFrom{ + Path: "/tmp/output.txt", + }, + }, + }, + }, + }) + } + + workflowObject := &v1alpha1.Workflow{ + ObjectMeta: v1.ObjectMeta{ + Name: req.TaskId, + Labels: map[string]string{ + "workflow-scope": "csghub", + }, + }, + Spec: v1alpha1.WorkflowSpec{ + Priority: ptr.To(int32(3)), + ServiceAccountName: config.Argo.ServiceAccountName, + Templates: templates, + Entrypoint: req.Entrypoint, + TTLStrategy: &v1alpha1.TTLStrategy{ + // Set TTL here + SecondsAfterCompletion: ptr.To(int32(config.Argo.JobTTL)), + }, + }, + } + + return workflowObject +} + +func (wc *workFlowComponentImpl) RunWorkflowsInformer(clusterPool *cluster.ClusterPool, c *config.Config) { + clientset := clusterPool.Clusters[0].ArgoClient + f := externalversions.NewSharedInformerFactoryWithOptions(clientset, 60*time.Second, externalversions.WithTweakListOptions(func(list *v1.ListOptions) { + list.LabelSelector = "workflow-scope=csghub" + })) + informer := f.Argoproj().V1alpha1().Workflows().Informer() + + stopper := make(chan struct{}) + defer close(stopper) + + defer runtime.HandleCrash() + + eventHandler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + // triggered in startup + wf := obj.(*v1alpha1.Workflow) + bg, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := wc.UpdateWorkflow(bg, wf) + if err != nil { + slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", wf.Name)) + } + }, + UpdateFunc: func(oldObj, newObj interface{}) { + oldWF := oldObj.(*v1alpha1.Workflow) + newWF := newObj.(*v1alpha1.Workflow) + + // compare status + bg, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if newWF.Status.Nodes == nil || oldWF.Status.Nodes == nil { + return + } + if oldWF.Status.Nodes[oldWF.Name].Phase != newWF.Status.Nodes[oldWF.Name].Phase { + _, err := wc.UpdateWorkflow(bg, newWF) + if err != nil { + slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", newWF.Name)) + } + } + }, + DeleteFunc: func(obj interface{}) { + //handle some special case + wf := obj.(*v1alpha1.Workflow) + bg, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := wc.DeleteWorkflowInargo(bg, wf) + if err != nil { + slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", wf.Name)) + } + }, + } + + _, _ = informer.AddEventHandler(eventHandler) + + informer.Run(stopper) + if !cache.WaitForCacheSync(stopper, informer.HasSynced) { + runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync")) + return + } +} + +func (wc *workFlowComponentImpl) StartAcctRequestFee(wf database.ArgoWorkflow) { + if !wc.config.IsMasterHost { + return + } + if wf.ResourceId == 0 { + return + } + duration := wf.EndTime.Sub(wf.StartTime) + minutes := duration.Minutes() + if minutes < 1 { + return + } + slog.Info("start to acct request fee", slog.Any("mins", minutes)) + event := types.METERING_EVENT{ + Uuid: uuid.New(), + UserUUID: wf.UserUUID, + Value: int64(minutes), + ValueType: types.TimeDurationMinType, + Scene: int(types.SceneEvaluation), + OpUID: "", + ResourceID: strconv.FormatInt(wf.ResourceId, 10), + ResourceName: wf.ResourceName, + CustomerID: wf.TaskId, + CreatedAt: time.Now(), + } + str, err := json.Marshal(event) + if err != nil { + slog.Error("error marshal metering event", slog.Any("event", event), slog.Any("error", err)) + return + } + err = wc.eventPub.PublishMeteringEvent(str) + if err != nil { + slog.Error("failed to pub metering event", slog.Any("data", string(str)), slog.Any("error", err)) + } else { + slog.Info("pub metering event success", slog.Any("data", string(str))) + } +} + +// get cluster +func GetCluster(ctx context.Context, clusterPool *cluster.ClusterPool, clusterID string) (*cluster.Cluster, string, error) { + if clusterID == "" { + clusterInfo, err := clusterPool.ClusterStore.ByClusterConfig(ctx, clusterPool.Clusters[0].ID) + if err != nil { + return nil, "", fmt.Errorf("failed to get cluster info: %v", err) + } + return &clusterPool.Clusters[0], clusterInfo.ClusterID, nil + } + cluster, err := clusterPool.GetClusterByID(ctx, clusterID) + if err != nil { + return nil, clusterID, fmt.Errorf("failed to get cluster by id: %v", err) + } + return cluster, clusterID, nil +} diff --git a/servicerunner/handler/k8s.go b/runner/handler/service.go similarity index 93% rename from servicerunner/handler/k8s.go rename to runner/handler/service.go index 15f08164..863e4c28 100644 --- a/servicerunner/handler/k8s.go +++ b/runner/handler/service.go @@ -21,10 +21,10 @@ import ( "opencsg.com/csghub-server/builder/store/database" "opencsg.com/csghub-server/common/config" "opencsg.com/csghub-server/common/types" - "opencsg.com/csghub-server/servicerunner/component" + "opencsg.com/csghub-server/runner/component" ) -type K8sHander struct { +type K8sHandler struct { clusterPool *cluster.ClusterPool k8sNameSpace string modelDockerRegBase string @@ -32,15 +32,10 @@ type K8sHander struct { s *component.ServiceComponent } -func NewK8sHander(config *config.Config) (*K8sHander, error) { - clusterPool, err := cluster.NewClusterPool() - if err != nil { - slog.Error("falied to build kubeconfig", "error", err) - return nil, fmt.Errorf("failed to build kubeconfig,%w", err) - } +func NewK8sHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*K8sHandler, error) { domainParts := strings.SplitN(config.Space.InternalRootDomain, ".", 2) serviceComponent := component.NewServiceComponent(config, domainParts[0]) - return &K8sHander{ + return &K8sHandler{ k8sNameSpace: domainParts[0], clusterPool: clusterPool, env: config, @@ -49,7 +44,7 @@ func NewK8sHander(config *config.Config) (*K8sHander, error) { }, nil } -func (s *K8sHander) RunService(c *gin.Context) { +func (s *K8sHandler) RunService(c *gin.Context) { request := &types.SVCRequest{} err := c.BindJSON(&request) if err != nil { @@ -148,7 +143,7 @@ func (s *K8sHander) RunService(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"message": "Service created successfully"}) } -func (s *K8sHander) StopService(c *gin.Context) { +func (s *K8sHandler) StopService(c *gin.Context) { var resp types.StopResponse var request = &types.StopRequest{} err := c.BindJSON(request) @@ -209,7 +204,7 @@ func (s *K8sHander) StopService(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) removeServiceForcely(c *gin.Context, cluster *cluster.Cluster, svcName string) error { +func (s *K8sHandler) removeServiceForcely(c *gin.Context, cluster *cluster.Cluster, svcName string) error { err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Delete(context.Background(), svcName, *metav1.NewDeleteOptions(0)) if err != nil { return err @@ -235,7 +230,7 @@ func (s *K8sHander) removeServiceForcely(c *gin.Context, cluster *cluster.Cluste return nil } -func (s *K8sHander) UpdateService(c *gin.Context) { +func (s *K8sHandler) UpdateService(c *gin.Context) { var resp types.ModelUpdateResponse var request = &types.ModelUpdateRequest{} err := c.BindJSON(request) @@ -294,7 +289,7 @@ func (s *K8sHander) UpdateService(c *gin.Context) { } // Update CPU and Memory requests and limits hardware := request.Hardware - resReq, _ := s.s.GenerateResources(hardware) + resReq, _ := component.GenerateResources(hardware) resources := corev1.ResourceRequirements{ Limits: resReq, Requests: resReq, @@ -320,7 +315,7 @@ func (s *K8sHander) UpdateService(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) ServiceStatus(c *gin.Context) { +func (s *K8sHandler) ServiceStatus(c *gin.Context) { var resp types.StatusResponse var request = &types.StatusRequest{} @@ -422,7 +417,7 @@ func (s *K8sHander) ServiceStatus(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) ServiceLogs(c *gin.Context) { +func (s *K8sHandler) ServiceLogs(c *gin.Context) { var request = &types.LogsRequest{} err := c.BindJSON(request) @@ -452,7 +447,7 @@ func (s *K8sHander) ServiceLogs(c *gin.Context) { s.GetLogsByPod(c, *cluster, podNames[0], srvName) } -func (s *K8sHander) ServiceLogsByPod(c *gin.Context) { +func (s *K8sHandler) ServiceLogsByPod(c *gin.Context) { var request = &types.ServiceRequest{} err := c.BindJSON(request) @@ -472,7 +467,7 @@ func (s *K8sHander) ServiceLogsByPod(c *gin.Context) { s.GetLogsByPod(c, *cluster, podName, srvName) } -func (s *K8sHander) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podName string, srvName string) { +func (s *K8sHandler) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podName string, srvName string) { logs := cluster.Client.CoreV1().Pods(s.k8sNameSpace).GetLogs(podName, &corev1.PodLogOptions{ Container: "user-container", @@ -536,14 +531,14 @@ func (s *K8sHander) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podNam slog.Error("write data failed", "error", err) } c.Writer.Flush() - slog.Info("send pod logs", slog.String("srv_name", srvName), slog.String("srv_name", srvName), slog.Int("len", n), slog.String("log", string(buf[:n]))) + slog.Info("send pod logs", slog.String("srv_name", srvName), slog.String("srv_name", srvName), slog.Int("len", n)) } } } } -func (s *K8sHander) ServiceStatusAll(c *gin.Context) { +func (s *K8sHandler) ServiceStatusAll(c *gin.Context) { allStatus := make(map[string]*types.StatusResponse) for index := range s.clusterPool.Clusters { cluster := s.clusterPool.Clusters[index] @@ -603,7 +598,7 @@ func (s *K8sHander) ServiceStatusAll(c *gin.Context) { c.JSON(http.StatusOK, allStatus) } -func (s *K8sHander) GetServicePods(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string, limit int64) ([]string, error) { +func (s *K8sHandler) GetServicePods(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string, limit int64) ([]string, error) { labelSelector := fmt.Sprintf("serving.knative.dev/service=%s", srvName) // Get the list of Pods based on the label selector opts := metav1.ListOptions{ @@ -629,12 +624,13 @@ func (s *K8sHander) GetServicePods(ctx context.Context, cluster cluster.Cluster, return podNames, nil } -func (s *K8sHander) GetClusterInfo(c *gin.Context) { +func (s *K8sHandler) GetClusterInfo(c *gin.Context) { clusterRes := []types.CluserResponse{} for index := range s.clusterPool.Clusters { cls := s.clusterPool.Clusters[index] - cInfo, _ := s.clusterPool.ClusterStore.ByClusterConfig(c.Request.Context(), cls.ID) - if !cInfo.Enable { + cInfo, err := s.clusterPool.ClusterStore.ByClusterConfig(c.Request.Context(), cls.ID) + if err != nil { + slog.Error("get cluster info failed", slog.Any("error", err)) continue } clusterInfo := types.CluserResponse{} @@ -649,7 +645,7 @@ func (s *K8sHander) GetClusterInfo(c *gin.Context) { c.JSON(http.StatusOK, clusterRes) } -func (s *K8sHander) GetClusterInfoByID(c *gin.Context) { +func (s *K8sHandler) GetClusterInfoByID(c *gin.Context) { clusterId := c.Params.ByName("id") cInfo, _ := s.clusterPool.ClusterStore.ByClusterID(c.Request.Context(), clusterId) clusterInfo := types.CluserResponse{} @@ -672,15 +668,15 @@ func (s *K8sHander) GetClusterInfoByID(c *gin.Context) { c.JSON(http.StatusOK, clusterInfo) } -func (s *K8sHander) getServiceNameFromRequest(c *gin.Context) string { +func (s *K8sHandler) getServiceNameFromRequest(c *gin.Context) string { return c.Params.ByName("service") } -func (s *K8sHander) getPodNameFromRequest(c *gin.Context) string { +func (s *K8sHandler) getPodNameFromRequest(c *gin.Context) string { return c.Params.ByName("pod_name") } -func (s *K8sHander) GetServiceByName(c *gin.Context) { +func (s *K8sHandler) GetServiceByName(c *gin.Context) { var resp types.StatusResponse var request = &types.CheckRequest{} err := c.BindJSON(request) @@ -740,7 +736,7 @@ func (s *K8sHander) GetServiceByName(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) GetReplica(c *gin.Context) { +func (s *K8sHandler) GetReplica(c *gin.Context) { var resp types.ReplicaResponse var request = &types.StatusRequest{} err := c.BindJSON(request) @@ -797,9 +793,6 @@ func (s *K8sHander) GetReplica(c *gin.Context) { } // revision exist - deployIDStr := srv.Annotations[types.ResDeployID] - deployID, _ := strconv.ParseInt(deployIDStr, 10, 64) - resp.DeployID = deployID resp.Code = 1 resp.Message = srvName resp.ActualReplica = int(*revision.Status.ActualReplicas) @@ -808,7 +801,7 @@ func (s *K8sHander) GetReplica(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) UpdateCluster(c *gin.Context) { +func (s *K8sHandler) UpdateCluster(c *gin.Context) { var resp types.UpdateClusterResponse var request = &database.ClusterInfo{} err := c.BindJSON(request) @@ -832,7 +825,7 @@ func (s *K8sHander) UpdateCluster(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) PurgeService(c *gin.Context) { +func (s *K8sHandler) PurgeService(c *gin.Context) { var resp types.PurgeResponse var request = &types.PurgeRequest{} err := c.BindJSON(request) @@ -890,7 +883,7 @@ func (s *K8sHander) PurgeService(c *gin.Context) { c.JSON(http.StatusOK, resp) } -func (s *K8sHander) GetServiceInfo(c *gin.Context) { +func (s *K8sHandler) GetServiceInfo(c *gin.Context) { var resp types.ServiceInfoResponse var request = &types.ServiceRequest{} err := c.BindJSON(request) diff --git a/runner/handler/workflow.go b/runner/handler/workflow.go new file mode 100644 index 00000000..b29f7b41 --- /dev/null +++ b/runner/handler/workflow.go @@ -0,0 +1,153 @@ +package handler + +import ( + "log/slog" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "opencsg.com/csghub-server/api/httpbase" + "opencsg.com/csghub-server/builder/deploy/cluster" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/common/types" + "opencsg.com/csghub-server/common/utils/common" + "opencsg.com/csghub-server/runner/component" +) + +type ArgoHandler struct { + clusterPool *cluster.ClusterPool + workflowNameSpace string + modelDockerRegBase string + config *config.Config + wfc component.WorkFlowComponent +} + +func NewArgoHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*ArgoHandler, error) { + wfc := component.NewWorkFlowComponent(config, clusterPool) + return &ArgoHandler{ + clusterPool: clusterPool, + config: config, + wfc: wfc, + modelDockerRegBase: config.Model.DockerRegBase, + workflowNameSpace: config.Argo.Namespace, + }, nil +} + +// create workflow +func (a *ArgoHandler) CreateWorkflow(ctx *gin.Context) { + var req types.ArgoWorkFlowReq + err := ctx.ShouldBindJSON(&req) + if err != nil { + slog.Error("bad order request format", "error", err) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + wf, err := a.wfc.CreateWorkflow(ctx, req) + if err != nil { + slog.Error("fail to create workflow", slog.Any("error", err), slog.Any("req", req)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + res := types.ArgoWorkFlowRes{ + ID: wf.ID, + Username: wf.Username, + TaskName: wf.TaskName, + TaskId: wf.TaskId, + TaskType: wf.TaskType, + TaskDesc: wf.TaskDesc, + RepoIds: wf.RepoIds, + RepoType: wf.RepoType, + SubmitTime: wf.SubmitTime, + } + + ctx.JSON(http.StatusOK, res) +} + +// list workflows +func (a *ArgoHandler) ListWorkflows(ctx *gin.Context) { + username := ctx.Query("username") + per, page, err := common.GetPerAndPageFromContext(ctx) + if err != nil { + slog.Error("Bad request format of page and per", slog.Any("error", err)) + httpbase.BadRequest(ctx, err.Error()) + return + } + wfs, total, err := a.wfc.FindWorkFlows(ctx, username, per, page) + if err != nil { + slog.Error("fail to list workflows", slog.Any("error", err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + ctx.JSON(http.StatusOK, gin.H{ + "total": total, + "list": wfs, + }) +} + +func (a *ArgoHandler) DeleteWorkflow(ctx *gin.Context) { + id := ctx.Param("id") + var req = &types.ArgoWorkFlowDeleteReq{} + err := ctx.BindJSON(req) + if err != nil { + slog.Error("bad request format", "error", err) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + idInt64, err := strconv.ParseInt(id, 10, 64) + if err != nil { + slog.Error("fail to convert id to int64", slog.Any("error", err), slog.Any("id", id)) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + err = a.wfc.DeleteWorkflow(ctx, idInt64, req.Username) + if err != nil { + slog.Error("fail to delete workflow", slog.Any("error", err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + slog.Info("Deleted argo workflow successfully", slog.String("id", id)) + httpbase.OK(ctx, nil) +} + +func (a *ArgoHandler) GetWorkflow(ctx *gin.Context) { + id := ctx.Param("id") + var req = &types.ArgoWorkFlowGetReq{} + err := ctx.BindJSON(req) + if err != nil { + slog.Error("bad request format", "error", err) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + idInt64, err := strconv.ParseInt(id, 10, 64) + if err != nil { + slog.Error("fail to convert id to int64", slog.Any("error", err), slog.Any("id", id)) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + wf, err := a.wfc.GetWorkflow(ctx, idInt64, req.Username) + if err != nil { + slog.Error("fail to get workflow", slog.Any("error", err)) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + res := types.ArgoWorkFlowRes{ + ID: wf.ID, + Username: wf.Username, + TaskName: wf.TaskName, + Image: wf.Image, + TaskId: wf.TaskId, + TaskType: wf.TaskType, + TaskDesc: wf.TaskDesc, + RepoIds: wf.RepoIds, + RepoType: wf.RepoType, + SubmitTime: wf.SubmitTime, + StartTime: wf.StartTime, + EndTime: wf.EndTime, + Datasets: wf.Datasets, + ResultURL: wf.ResultURL, + DownloadURL: wf.DownloadURL, + FailuresURL: wf.FailuresURL, + Status: wf.Status, + } + ctx.JSON(http.StatusOK, res) +} diff --git a/runner/router/api.go b/runner/router/api.go new file mode 100644 index 00000000..9a532d28 --- /dev/null +++ b/runner/router/api.go @@ -0,0 +1,65 @@ +package router + +import ( + "fmt" + "log/slog" + + "github.com/gin-gonic/gin" + "opencsg.com/csghub-server/api/middleware" + "opencsg.com/csghub-server/builder/deploy/cluster" + "opencsg.com/csghub-server/common/config" + "opencsg.com/csghub-server/runner/handler" +) + +func NewHttpServer(config *config.Config) (*gin.Engine, error) { + r := gin.New() + r.Use(gin.Recovery()) + r.Use(middleware.Log()) + + clusterPool, err := cluster.NewClusterPool() + if err != nil { + slog.Error("falied to build kubeconfig", "error", err) + return nil, fmt.Errorf("failed to build kubeconfig,%w", err) + } + + k8sHandler, err := handler.NewK8sHandler(config, clusterPool) + if err != nil { + return nil, fmt.Errorf("failed to build NewHttpServer,%w", err) + } + apiGroup := r.Group("/api/v1") + service := apiGroup.Group("/service") + { + service.POST("/:service/run", k8sHandler.RunService) + service.PUT("/:service/update", k8sHandler.UpdateService) + service.POST("/:service/stop", k8sHandler.StopService) + service.GET("/:service/status", k8sHandler.ServiceStatus) + service.GET("/:service/logs", k8sHandler.ServiceLogs) + service.GET("/:service/logs/:pod_name", k8sHandler.ServiceLogsByPod) + service.GET("/:service/info", k8sHandler.GetServiceInfo) + service.GET("/status-all", k8sHandler.ServiceStatusAll) + service.GET("/:service/get", k8sHandler.GetServiceByName) + service.GET("/:service/replica", k8sHandler.GetReplica) + service.DELETE("/:service/purge", k8sHandler.PurgeService) + + } + cluster := apiGroup.Group("/cluster") + { + cluster.GET("", k8sHandler.GetClusterInfo) + cluster.GET("/:id", k8sHandler.GetClusterInfoByID) + cluster.PUT("/:id", k8sHandler.UpdateCluster) + } + argoHandler, err := handler.NewArgoHandler(config, clusterPool) + if err != nil { + return nil, fmt.Errorf("failed to build NewHttpServer,%w", err) + } + + workflows := apiGroup.Group("/workflows") + { + workflows.POST("", argoHandler.CreateWorkflow) + workflows.GET("", argoHandler.ListWorkflows) + workflows.DELETE("/:id", argoHandler.DeleteWorkflow) + workflows.GET("/:id", argoHandler.GetWorkflow) + } + + return r, nil +} diff --git a/servicerunner/router/api.go b/servicerunner/router/api.go deleted file mode 100644 index 2c5153e0..00000000 --- a/servicerunner/router/api.go +++ /dev/null @@ -1,45 +0,0 @@ -package router - -import ( - "fmt" - - "github.com/gin-gonic/gin" - "opencsg.com/csghub-server/api/middleware" - "opencsg.com/csghub-server/common/config" - "opencsg.com/csghub-server/servicerunner/handler" -) - -func NewHttpServer(config *config.Config) (*gin.Engine, error) { - r := gin.New() - r.Use(gin.Recovery()) - r.Use(middleware.Log()) - - handler, err := handler.NewK8sHander(config) - if err != nil { - return nil, fmt.Errorf("failed to build NewHttpServer,%w", err) - } - apiGroup := r.Group("/api/v1") - service := apiGroup.Group("/service") - { - service.POST("/:service/run", handler.RunService) - service.PUT("/:service/update", handler.UpdateService) - service.POST("/:service/stop", handler.StopService) - service.GET("/:service/status", handler.ServiceStatus) - service.GET("/:service/logs", handler.ServiceLogs) - service.GET("/:service/logs/:pod_name", handler.ServiceLogsByPod) - service.GET("/:service/info", handler.GetServiceInfo) - service.GET("/status-all", handler.ServiceStatusAll) - service.GET("/:service/get", handler.GetServiceByName) - service.GET("/:service/replica", handler.GetReplica) - service.DELETE("/:service/purge", handler.PurgeService) - - } - cluster := apiGroup.Group("/cluster") - { - cluster.GET("", handler.GetClusterInfo) - cluster.GET("/:id", handler.GetClusterInfoByID) - cluster.PUT("/:id", handler.UpdateCluster) - } - - return r, nil -}