+
+
package component
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingBillComponentImpl struct {
+ abs database.AccountBillStore
+}
+
+type AccountingBillComponent interface {
+ ListBillsByUserIDAndDate(ctx context.Context, req types.ACCT_BILLS_REQ) (database.AccountBillRes, error)
+}
+
+func NewAccountingBillComponent() AccountingBillComponent {
+ abc := &accountingBillComponentImpl{
+ abs: database.NewAccountBillStore(),
+ }
+ return abc
+}
+
+func (a *accountingBillComponentImpl) ListBillsByUserIDAndDate(ctx context.Context, req types.ACCT_BILLS_REQ) (database.AccountBillRes, error) {
+ return a.abs.ListByUserIDAndDate(ctx, req)
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "reflect"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingEventComponentImpl struct {
+ ae database.AccountEventStore
+}
+
+type AccountingEventComponent interface {
+ AddNewAccountingEvent(ctx context.Context, event *types.AcctEvent) error
+}
+
+func NewAccountingEventComponent() AccountingEventComponent {
+ aec := &accountingEventComponentImpl{
+ ae: database.NewAccountEventStore(),
+ }
+ return aec
+}
+
+func (a *accountingEventComponentImpl) AddNewAccountingEvent(ctx context.Context, event *types.AcctEvent) error {
+ _, err := a.ae.GetByEventID(ctx, event.Uuid)
+ if errors.Is(err, sql.ErrNoRows) {
+ body := make(map[string]string)
+ elem := reflect.ValueOf(event).Elem()
+ relType := elem.Type()
+ for i := 0; i < relType.NumField(); i++ {
+ name := relType.Field(i).Name
+ body[name] = fmt.Sprintf("%v", elem.Field(i).Interface())
+ }
+ input := database.AccountEvent{
+ EventUUID: event.Uuid,
+ EventBody: body,
+ }
+ return a.ae.Create(ctx, input)
+ }
+
+ return err
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+
+ "opencsg.com/csghub-server/accounting/utils"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type meteringComponentImpl struct {
+ ams database.AccountMeteringStore
+}
+
+type MeteringComponent interface {
+ SaveMeteringEventRecord(ctx context.Context, req *types.METERING_EVENT) error
+ ListMeteringByUserIDAndDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]database.AccountMetering, int, error)
+ GetMeteringStatByDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]map[string]interface{}, error)
+}
+
+func NewMeteringComponent() MeteringComponent {
+ ams := &meteringComponentImpl{
+ ams: database.NewAccountMeteringStore(),
+ }
+ return ams
+}
+
+func (mc *meteringComponentImpl) SaveMeteringEventRecord(ctx context.Context, req *types.METERING_EVENT) error {
+ am := database.AccountMetering{
+ EventUUID: req.Uuid,
+ UserUUID: req.UserUUID,
+ Value: float64(req.Value),
+ ValueType: req.ValueType,
+ Scene: types.SceneType(req.Scene),
+ OpUID: req.OpUID,
+ ResourceID: req.ResourceID,
+ ResourceName: req.ResourceName,
+ CustomerID: req.CustomerID,
+ RecordedAt: req.CreatedAt,
+ Extra: req.Extra,
+ SkuUnitType: utils.GetSkuUnitTypeByScene(types.SceneType(req.Scene)),
+ }
+ err := mc.ams.Create(ctx, am)
+ if err != nil {
+ return fmt.Errorf("failed to save metering event record, error: %w", err)
+ }
+ return nil
+}
+
+func (mc *meteringComponentImpl) ListMeteringByUserIDAndDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]database.AccountMetering, int, error) {
+ meters, total, err := mc.ams.ListByUserIDAndTime(ctx, req)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to list metering by UserIDAndDate, error: %w", err)
+ }
+ return meters, total, nil
+}
+
+func (mc *meteringComponentImpl) GetMeteringStatByDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]map[string]interface{}, error) {
+ res, err := mc.ams.GetStatByDate(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get metering stat, error: %w", err)
+ }
+ return res, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingOrderComponentImpl struct {
+ aos database.AccountOrderStore
+ aps database.AccountPriceStore
+}
+
+type AccountingOrderComponent interface {
+ Create(ctx context.Context, req types.AcctOrderCreateReq, nowTime time.Time) (*database.AccountOrder, error)
+ GetByID(ctx context.Context, orderUUID string) (*database.AccountOrder, error)
+ GetDetailByID(ctx context.Context, id int64) (*database.AccountOrderDetail, error)
+}
+
+func NewAccountingOrderComponent() AccountingOrderComponent {
+ aoc := &accountingOrderComponentImpl{
+ aos: database.NewAccountOrderStore(),
+ aps: database.NewAccountPriceStore(),
+ }
+ return aoc
+}
+
+func (a *accountingOrderComponentImpl) Create(ctx context.Context, req types.AcctOrderCreateReq, nowTime time.Time) (*database.AccountOrder, error) {
+ // now := time.Now()
+ order := database.AccountOrder{
+ OrderUUID: req.OrderUUID.String(),
+ UserUUID: req.UserUUID,
+ OrderStatus: types.OrderPaid,
+ }
+ var orderDetails []database.AccountOrderDetail
+ orderAmount := 0.0
+ for _, detail := range req.OrderDetails {
+ isUnitType := a.verifySkuUnitType(detail.SkuUnitType)
+ if !isUnitType {
+ return nil, fmt.Errorf("invalid unit type %s found", detail.SkuUnitType)
+ }
+ price, err := a.aps.GetLatestByTime(ctx, types.AcctPriceQueryReq{
+ SkuType: detail.SkuType,
+ ResourceID: detail.ResourceID,
+ SkuKind: types.SKUTimeSpan,
+ SkuUnitType: detail.SkuUnitType,
+ PriceTime: nowTime,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("fail to get price for resource %s, sku type %d: %w", detail.ResourceID, detail.SkuType, err)
+ }
+ beginTime, endTime, err := a.getTimeSpan(detail, price)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get time span for resource %s, sku type %d: %w", detail.ResourceID, detail.SkuType, err)
+ }
+ amount := 0 - float64(price.SkuPrice*int64(detail.OrderCount))
+ orderDetails = append(orderDetails, database.AccountOrderDetail{
+ OrderUUID: req.OrderUUID.String(),
+ SkuType: detail.SkuType,
+ ResourceID: detail.ResourceID,
+ SkuKind: types.SKUTimeSpan,
+ SkuUnitType: detail.SkuUnitType,
+ OrderCount: detail.OrderCount,
+ SkuPriceID: price.ID,
+ Amount: amount,
+ BeginTime: *beginTime,
+ EndTime: *endTime,
+ })
+ orderAmount += amount
+ }
+ order.Amount = orderAmount
+ order.Details = orderDetails
+
+ statement := database.AccountStatement{
+ EventUUID: req.OrderUUID,
+ UserUUID: req.UserUUID,
+ Value: orderAmount,
+ Scene: types.ScenePayOrder,
+ OpUID: req.UserUUID,
+ CustomerID: "",
+ EventDate: nowTime,
+ Price: 0,
+ PriceUnit: "",
+ Consumption: 0,
+ ValueType: 0,
+ ResourceID: "",
+ ResourceName: "",
+ SkuID: 0,
+ RecordedAt: nowTime,
+ SkuUnit: 0,
+ SkuUnitType: "",
+ SkuPriceCurrency: "",
+ IsCancel: false,
+ EventValue: orderAmount,
+ }
+
+ err := a.aos.Create(ctx, order, statement)
+ if err != nil {
+ return nil, fmt.Errorf("fail to build order and details, %w", err)
+ }
+ return &order, nil
+}
+
+func (a *accountingOrderComponentImpl) GetByID(ctx context.Context, orderUUID string) (*database.AccountOrder, error) {
+ order, err := a.aos.GetByID(ctx, orderUUID)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get order by uuid, %w", err)
+ }
+ return order, nil
+}
+
+func (a *accountingOrderComponentImpl) getTimeSpan(reqDetail types.AcctOrderDetailReq, price *database.AccountPrice) (*time.Time, *time.Time, error) {
+ var (
+ err error
+ beginTime = time.Now()
+ endTime = time.Now()
+ )
+ if len(reqDetail.BeginTime) > 0 {
+ beginTime, err = time.Parse("2006-01-02", reqDetail.BeginTime)
+ if err != nil {
+ return nil, nil, fmt.Errorf("fail to parse begin_time: %w", err)
+ }
+ }
+ span := int(price.SkuUnit) * reqDetail.OrderCount
+
+ if price.SkuUnitType == types.UnitMonth {
+ endTime = beginTime.AddDate(0, span, 0)
+ } else if price.SkuUnitType == types.UnitYear {
+ endTime = beginTime.AddDate(span, 0, 0)
+ } else {
+ endTime = beginTime.Add(time.Duration(span) * time.Minute)
+ }
+ endTime = endTime.Truncate(24 * time.Hour)
+ return &beginTime, &endTime, nil
+}
+
+func (a *accountingOrderComponentImpl) GetDetailByID(ctx context.Context, id int64) (*database.AccountOrderDetail, error) {
+ detail, err := a.aos.GetDetailByID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get order by uuid, %w", err)
+ }
+ return detail, nil
+}
+
+func (a *accountingOrderComponentImpl) verifySkuUnitType(unitType string) bool {
+ return unitType == types.UnitMonth || unitType == types.UnitYear
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingPresentComponentImpl struct {
+ aps database.AccountPresentStore
+}
+
+type AccountingPresentComponent interface {
+ PresentAccountingUser(ctx context.Context, userUUID string, req types.ACTIVITY_REQ) error
+}
+
+func NewAccountingPresentComponent() AccountingPresentComponent {
+ apm := &accountingPresentComponentImpl{
+ aps: database.NewAccountPresentStore(),
+ }
+ return apm
+}
+
+func (a *accountingPresentComponentImpl) PresentAccountingUser(ctx context.Context, userUUID string, req types.ACTIVITY_REQ) error {
+ present, err := a.aps.FindPresentByUserIDAndScene(ctx, userUUID, req.ID)
+ if err != nil {
+ return fmt.Errorf("fail to find present, error: %w", err)
+ }
+ slog.Debug("PresentAccountingUser", slog.Any("present", present))
+ if present != nil {
+ slog.Warn("duplicated send present for account", slog.Any("userUUID", userUUID), slog.Any("req", req))
+ return nil
+ }
+ eventUUID := uuid.New()
+ input := database.AccountPresent{
+ EventUUID: eventUUID,
+ UserUUID: userUUID,
+ ActivityID: req.ID,
+ Value: req.Value,
+ OpUID: req.OpUID,
+ OpDesc: req.OpDesc,
+ }
+
+ statement := database.AccountStatement{
+ EventUUID: eventUUID,
+ UserUUID: userUUID,
+ Value: req.Value,
+ Scene: types.ScenePortalCharge,
+ OpUID: req.OpUID,
+ CustomerID: "",
+ EventDate: time.Now(),
+ Price: 0,
+ PriceUnit: "",
+ Consumption: 0,
+ ValueType: 0,
+ ResourceID: "",
+ ResourceName: "",
+ SkuID: 0,
+ RecordedAt: time.Now(),
+ SkuUnit: 0,
+ SkuUnitType: "",
+ SkuPriceCurrency: "",
+ }
+ err = a.aps.AddPresent(ctx, input, statement)
+ if err != nil {
+ return fmt.Errorf("fail to add present and change balance, error: %w", err)
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingPriceComponentImpl struct {
+ au database.AccountPriceStore
+}
+
+type AccountingPriceComponent interface {
+ GetPriceByID(ctx context.Context, id int64) (*database.AccountPrice, error)
+ CreatePrice(ctx context.Context, req types.AcctPriceCreateReq) (*database.AccountPrice, error)
+ UpdatePrice(ctx context.Context, req types.AcctPriceCreateReq, id int64) (*database.AccountPrice, error)
+ DeletePrice(ctx context.Context, id int64) error
+ GetLatestByTime(ctx context.Context, req types.AcctPriceQueryReq) (*database.AccountPrice, error)
+ ListPricesBySKUType(ctx context.Context, req types.AcctPriceListReq) ([]database.AccountPrice, int, error)
+}
+
+func NewAccountingPriceComponent() AccountingPriceComponent {
+ apc := &accountingPriceComponentImpl{
+ au: database.NewAccountPriceStore(),
+ }
+ return apc
+}
+
+func (a *accountingPriceComponentImpl) GetPriceByID(ctx context.Context, id int64) (*database.AccountPrice, error) {
+ price, err := a.au.GetByID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get price by id, %w", err)
+ }
+ return price, nil
+}
+
+func (a *accountingPriceComponentImpl) CreatePrice(ctx context.Context, req types.AcctPriceCreateReq) (*database.AccountPrice, error) {
+ price := database.AccountPrice{
+ SkuType: req.SkuType,
+ SkuPrice: req.SkuPrice,
+ SkuUnit: req.SkuUnit,
+ SkuDesc: req.SkuDesc,
+ ResourceID: req.ResourceID,
+ SkuUnitType: req.SkuUnitType,
+ SkuPriceCurrency: req.SkuPriceCurrency,
+ SkuKind: req.SkuKind,
+ Quota: req.Quota,
+ SkuPriceID: req.SkuPriceID,
+ }
+ res, err := a.au.Create(ctx, price)
+ if err != nil {
+ return nil, fmt.Errorf("failed to add price, error: %w", err)
+ }
+ return res, nil
+}
+
+func (a *accountingPriceComponentImpl) UpdatePrice(ctx context.Context, req types.AcctPriceCreateReq, id int64) (*database.AccountPrice, error) {
+ price := database.AccountPrice{
+ ID: id,
+ SkuType: req.SkuType,
+ SkuPrice: req.SkuPrice,
+ SkuUnit: req.SkuUnit,
+ SkuDesc: req.SkuDesc,
+ ResourceID: req.ResourceID,
+ SkuUnitType: req.SkuUnitType,
+ SkuPriceCurrency: req.SkuPriceCurrency,
+ SkuKind: req.SkuKind,
+ Quota: req.Quota,
+ SkuPriceID: req.SkuPriceID,
+ }
+ res, err := a.au.Update(ctx, price)
+ if err != nil {
+ return nil, fmt.Errorf("failed to modify price, %w", err)
+ }
+ return res, nil
+}
+
+func (a *accountingPriceComponentImpl) DeletePrice(ctx context.Context, id int64) error {
+ price := database.AccountPrice{
+ ID: id,
+ }
+ err := a.au.Delete(ctx, price)
+ if err != nil {
+ return fmt.Errorf("failed to remove price, %w", err)
+ }
+ return err
+}
+
+func (a *accountingPriceComponentImpl) GetLatestByTime(ctx context.Context, req types.AcctPriceQueryReq) (*database.AccountPrice, error) {
+ price, err := a.au.GetLatestByTime(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get price by time, %w", err)
+ }
+ return price, nil
+}
+
+func (a *accountingPriceComponentImpl) ListPricesBySKUType(ctx context.Context, req types.AcctPriceListReq) ([]database.AccountPrice, int, error) {
+ prices, total, err := a.au.ListBySkuType(ctx, req)
+ if err != nil {
+ return nil, 0, fmt.Errorf("list price by skutype/skukind/resourceid, %w", err)
+ }
+
+ return prices, total, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/common/utils/payment"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "time"
+)
+
+type RechargeComponent interface {
+ CreateRecharge(ctx context.Context, userUUID string, fromUserUUID string, description string, amount *money.Money, channel consts.PaymentChannel) (*database.AccountRecharge, *rpc.PaymentResponse, error)
+ RechargeSucceed(ctx context.Context, orderNo string) (*database.AccountRecharge, error)
+ RechargeClosed(ctx context.Context, orderNo string) (*database.AccountRecharge, error)
+}
+
+type rechargeComponentImpl struct {
+ ars database.AccountRechargeStore
+ paymentRpc rpc.PaymentSvcClient
+}
+
+func NewRechargeComponent(config *config.Config) RechargeComponent {
+ c := &rechargeComponentImpl{
+ ars: database.NewAccountRechargeStore(),
+ paymentRpc: rpc.NewPaymentSvcHttpClient(fmt.Sprintf("%s:%d", config.Payment.Host, config.Payment.Port)),
+ }
+ return c
+}
+
+func (c *rechargeComponentImpl) CreateRecharge(
+ ctx context.Context,
+ userUUID string,
+ fromUserUUID string,
+ description string,
+ amount *money.Money,
+ channel consts.PaymentChannel) (*database.AccountRecharge, *rpc.PaymentResponse, error) {
+ extra := types.RechargeExtraType
+ orderNo, err := payment.GenerateOrderNumber()
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to create payment for %s, error: %w", channel, err)
+ }
+ paymentObj, err := c.paymentRpc.CreateSimplePayment(ctx, orderNo, description, description, amount, extra, channel)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to create payment for %s, error: %w", channel, err)
+ }
+
+ recharge := &database.AccountRecharge{}
+ recharge.RechargeUUID = uuid.NewString()
+ recharge.OrderNo = paymentObj.OrderNo
+ recharge.PaymentUUID = paymentObj.PaymentUUID
+ recharge.UserUUID = userUUID
+ recharge.FromUserUUID = fromUserUUID
+ recharge.Amount = amount.GetAmount()
+ recharge.Description = description
+ recharge.Channel = channel
+
+ err = c.ars.CreateRecharge(ctx, recharge)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to create recharge, error: %w", err)
+ }
+ return recharge, paymentObj, nil
+}
+
+func (c *rechargeComponentImpl) RechargeSucceed(
+ ctx context.Context,
+ orderNo string,
+) (*database.AccountRecharge, error) {
+ recharge, err := c.ars.GetRechargeByOrderNo(ctx, orderNo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to success a recharge for %s, error: %w", orderNo, err)
+ }
+ if !recharge.Succeeded {
+ recharge.UpdatedAt = time.Now()
+ recharge.TimeSucceeded = time.Now()
+ recharge.Succeeded = true
+ er := c.ars.UpdateRecharge(ctx, recharge)
+ if er != nil {
+ return nil, fmt.Errorf("failed to success a recharge for %s, error: %w", orderNo, er)
+ }
+ }
+ return recharge, nil
+}
+
+func (c *rechargeComponentImpl) RechargeClosed(
+ ctx context.Context,
+ orderNo string,
+) (*database.AccountRecharge, error) {
+ recharge, err := c.ars.GetRechargeByOrderNo(ctx, orderNo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to fetch a recharge for %s, error: %w", orderNo, err)
+ }
+ if recharge.Succeeded {
+ return nil, fmt.Errorf("can not to close a succeed recharge for %s", orderNo)
+ }
+ if !recharge.Closed {
+ recharge.UpdatedAt = time.Now()
+ recharge.Closed = true
+ er := c.ars.UpdateRecharge(ctx, recharge)
+ if er != nil {
+ return nil, fmt.Errorf("failed to close a recharge for %s, error: %w", orderNo, er)
+ }
+ }
+ return recharge, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingStatementComponentImpl struct {
+ asms database.AccountStatementStore
+}
+
+type AccountingStatementComponent interface {
+ AddNewStatement(ctx context.Context, event *types.ACCT_EVENT_REQ) error
+ ListStatementByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (types.ACCT_STATEMENTS_RESULT, error)
+ FindStatementByEventID(ctx context.Context, event *types.AcctEvent) (*database.AccountStatement, error)
+ RechargeAccountingUser(ctx context.Context, userUUID string, req types.RECHARGE_REQ, eventUUID uuid.UUID) error
+}
+
+func NewAccountingStatementComponent() AccountingStatementComponent {
+ asc := &accountingStatementComponentImpl{
+ asms: database.NewAccountStatementStore(),
+ }
+ return asc
+}
+
+func (a *accountingStatementComponentImpl) AddNewStatement(ctx context.Context, event *types.ACCT_EVENT_REQ) error {
+ statement := database.AccountStatement{
+ EventUUID: event.EventUUID,
+ UserUUID: event.UserUUID,
+ Value: event.Value,
+ Scene: event.Scene,
+ OpUID: event.OpUID,
+ CustomerID: event.CustomerID,
+ EventDate: event.RecordedAt,
+ Price: event.Price,
+ PriceUnit: event.PriceUnit,
+ Consumption: event.Consumption,
+ ValueType: event.ValueType,
+ ResourceID: event.ResourceID,
+ ResourceName: event.ResourceName,
+ SkuID: event.SkuID,
+ RecordedAt: event.RecordedAt,
+ SkuUnit: event.SkuUnit,
+ SkuUnitType: event.SkuUnitType,
+ SkuPriceCurrency: event.SkuPriceCurrency,
+ EventValue: event.Value,
+ IsCancel: false,
+ }
+ err := a.asms.Create(ctx, statement)
+ if err != nil {
+ return fmt.Errorf("fail to save accounting statement, %w", err)
+ }
+ return nil
+}
+
+func (a *accountingStatementComponentImpl) ListStatementByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (types.ACCT_STATEMENTS_RESULT, error) {
+ statements, err := a.asms.ListByUserIDAndTime(ctx, req)
+ if err != nil {
+ return types.ACCT_STATEMENTS_RESULT{}, fmt.Errorf("fail to list accounting statement by user and time, %w", err)
+ }
+
+ var resStatements []types.ACCT_STATEMENTS_RES
+ for _, st := range statements.Data {
+ resStatements = append(resStatements, types.ACCT_STATEMENTS_RES{
+ ID: st.ID,
+ EventUUID: st.EventUUID,
+ UserUUID: st.UserUUID,
+ Value: st.Value,
+ Scene: int(st.Scene),
+ OpUID: st.OpUID,
+ CreatedAt: st.CreatedAt,
+ CustomerID: st.CustomerID,
+ EventDate: st.EventDate,
+ Price: st.Price,
+ PriceUnit: st.PriceUnit,
+ Consumption: st.Consumption,
+ SkuUnit: st.SkuUnit,
+ SkuUnitType: st.SkuUnitType,
+ SkuPriceCurrency: st.SkuPriceCurrency,
+ })
+ }
+
+ return types.ACCT_STATEMENTS_RESULT{Data: resStatements, ACCT_SUMMARY: statements.ACCT_SUMMARY}, err
+}
+
+func (a *accountingStatementComponentImpl) FindStatementByEventID(ctx context.Context, event *types.AcctEvent) (*database.AccountStatement, error) {
+ statement, err := a.asms.GetByEventID(ctx, event.Uuid)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("fail to find statement by event uuid, %w", err)
+ }
+ return &statement, nil
+}
+
+func (a *accountingStatementComponentImpl) RechargeAccountingUser(ctx context.Context, userUUID string, req types.RECHARGE_REQ, eventUUID uuid.UUID) error {
+ actionScene := types.ScenePortalCharge
+ if req.Scene == types.SceneCashCharge {
+ actionScene = types.SceneCashCharge
+ }
+ event := types.ACCT_EVENT_REQ{
+ EventUUID: eventUUID,
+ UserUUID: userUUID,
+ Value: req.Value,
+ Scene: actionScene,
+ OpUID: req.OpUID,
+ CustomerID: "",
+ EventDate: time.Now(),
+ Price: 0,
+ PriceUnit: "",
+ Consumption: 0,
+ ValueType: 0,
+ ResourceID: "",
+ ResourceName: "",
+ SkuID: 0,
+ RecordedAt: time.Now(),
+ }
+ err := a.AddNewStatement(ctx, &event)
+ if err != nil {
+ return fmt.Errorf("fail to add statement and rechange balance, %v, %w", event, err)
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingSyncQuotaComponentImpl struct {
+ asq database.AccountSyncQuotaStore
+ user database.UserStore
+}
+
+type AccountingSyncQuotaComponent interface {
+ GetQuotaByID(ctx context.Context, currentUser string) (*database.AccountSyncQuota, error)
+ CreateOrUpdateQuota(ctx context.Context, currentUser string, input types.ACCT_QUOTA_REQ) (*database.AccountSyncQuota, error)
+}
+
+func NewAccountingQuotaComponent() AccountingSyncQuotaComponent {
+ asqc := &accountingSyncQuotaComponentImpl{
+ asq: database.NewAccountSyncQuotaStore(),
+ user: database.NewUserStore(),
+ }
+ return asqc
+}
+
+func (a *accountingSyncQuotaComponentImpl) GetQuotaByID(ctx context.Context, currentUser string) (*database.AccountSyncQuota, error) {
+ user, err := a.user.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ acctQuota, err := a.asq.GetByID(ctx, user.ID)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("fail to get account quota, %w", err)
+ }
+ return acctQuota, nil
+}
+
+func (a *accountingSyncQuotaComponentImpl) CreateOrUpdateQuota(ctx context.Context, currentUser string, input types.ACCT_QUOTA_REQ) (*database.AccountSyncQuota, error) {
+ user, err := a.user.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ quota, err := a.asq.GetByID(ctx, user.ID)
+ if errors.Is(err, sql.ErrNoRows) {
+ // create account quota
+ acctQuota := database.AccountSyncQuota{
+ UserID: user.ID,
+ RepoCountLimit: input.RepoCountLimit,
+ RepoCountUsed: 0,
+ SpeedLimit: input.SpeedLimit,
+ TrafficLimit: input.TrafficLimit,
+ TrafficUsed: 0,
+ }
+ err := a.asq.Create(ctx, acctQuota)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create account quota, %w", err)
+ }
+ return &acctQuota, nil
+ }
+
+ if err != nil || quota == nil {
+ return nil, fmt.Errorf("fail to query account quota, %w", err)
+ }
+
+ // update account quota
+ quota.RepoCountLimit = input.RepoCountLimit
+ quota.SpeedLimit = input.SpeedLimit
+ quota.TrafficLimit = input.TrafficLimit
+ _, err = a.asq.Update(ctx, *quota)
+ if err != nil {
+ return nil, fmt.Errorf("fail to update account quota, %w", err)
+ }
+
+ return quota, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingSyncQuotaStatementComponentImpl struct {
+ asq database.AccountSyncQuotaStatementStore
+ user database.UserStore
+ repo database.RepoStore
+}
+
+type AccountingSyncQuotaStatementComponent interface {
+ CreateQuotaStatement(ctx context.Context, currentUser string, input types.ACCT_QUOTA_STATEMENT_REQ) (*database.AccountSyncQuotaStatement, error)
+ GetQuotaStatement(ctx context.Context, currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (*database.AccountSyncQuotaStatement, error)
+}
+
+func NewAccountingQuotaStatementComponent() AccountingSyncQuotaStatementComponent {
+ asqsc := &accountingSyncQuotaStatementComponentImpl{
+ asq: database.NewAccountSyncQuotaStatementStore(),
+ user: database.NewUserStore(),
+ repo: database.NewRepoStore(),
+ }
+ return asqsc
+}
+
+func (a *accountingSyncQuotaStatementComponentImpl) CreateQuotaStatement(ctx context.Context, currentUser string, input types.ACCT_QUOTA_STATEMENT_REQ) (*database.AccountSyncQuotaStatement, error) {
+ user, err := a.user.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ fields := strings.Split(input.RepoPath, "/")
+ if len(fields) != 2 {
+ return nil, errors.New("invalid repo")
+ }
+ r, err := a.repo.FindByPath(ctx, types.RepositoryType(input.RepoType), fields[0], fields[1])
+ if err != nil || r == nil {
+ return nil, errors.New("repo does not exist")
+ }
+
+ acctQuotaSM := database.AccountSyncQuotaStatement{
+ UserID: user.ID,
+ RepoPath: input.RepoPath,
+ RepoType: input.RepoType,
+ }
+
+ err = a.asq.Create(ctx, acctQuotaSM)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create quota statement, %w", err)
+ }
+ return &acctQuotaSM, nil
+}
+
+func (a *accountingSyncQuotaStatementComponentImpl) GetQuotaStatement(ctx context.Context, currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (*database.AccountSyncQuotaStatement, error) {
+ user, err := a.user.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ quotaSM, err := a.asq.Get(ctx, user.ID, req)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("fail to get quota statement, %w", err)
+ }
+ return quotaSM, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingUserComponentImpl struct {
+ au database.AccountUserStore
+}
+
+type AccountingUserComponent interface {
+ ListAccountingUser(ctx context.Context, per, page int) ([]database.AccountUser, int, error)
+ GetAccountingByUserID(ctx context.Context, userUUID string) (*types.UserBalanceResp, error)
+ AddNewAccountingUser(ctx context.Context, event *types.AcctEvent) error
+ CheckAccountingUser(ctx context.Context, userUUID string) error
+}
+
+func NewAccountingUserComponent() AccountingUserComponent {
+ auc := &accountingUserComponentImpl{
+ au: database.NewAccountUserStore(),
+ }
+ return auc
+}
+
+func (a *accountingUserComponentImpl) ListAccountingUser(ctx context.Context, per, page int) ([]database.AccountUser, int, error) {
+ return a.au.List(ctx, per, page)
+}
+
+func (a *accountingUserComponentImpl) GetAccountingByUserID(ctx context.Context, userUUID string) (*types.UserBalanceResp, error) {
+ account, err := a.au.FindUserByID(ctx, userUUID)
+ if errors.Is(err, sql.ErrNoRows) {
+ return &types.UserBalanceResp{
+ UserUUID: userUUID,
+ Balance: 0,
+ Composition: types.ComposedBalance{
+ Cash: 0,
+ Bonus: 0,
+ },
+ }, nil
+ }
+
+ resp := types.UserBalanceResp{
+ UserUUID: userUUID,
+ Balance: account.Balance + account.CashBalance,
+ Composition: types.ComposedBalance{
+ Cash: account.CashBalance,
+ Bonus: account.Balance,
+ },
+ }
+
+ return &resp, err
+}
+
+func (a *accountingUserComponentImpl) AddNewAccountingUser(ctx context.Context, event *types.AcctEvent) error {
+ statement := database.AccountUser{
+ UserUUID: event.UserUUID,
+ Balance: 0,
+ }
+ return a.au.Create(ctx, statement)
+}
+
+func (a *accountingUserComponentImpl) CheckAccountingUser(ctx context.Context, userUUID string) error {
+ _, err := a.au.FindUserByID(ctx, userUUID)
+ if errors.Is(err, sql.ErrNoRows) {
+ statement := database.AccountUser{
+ UserUUID: userUUID,
+ Balance: 0,
+ }
+ return a.au.Create(ctx, statement)
+ }
+ return err
+}
+
+
+
package consumer
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/nats-io/nats.go"
+ "github.com/nats-io/nats.go/jetstream"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/accounting/utils"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mq"
+)
+
+var (
+ idleDuration = 10 * time.Second
+)
+
+type Charging struct {
+ sysMQ mq.MessageQueue
+ acctSMComp component.AccountingStatementComponent
+ acctUserComp component.AccountingUserComponent
+ acctEvtComp component.AccountingEventComponent
+ acctPriceComp component.AccountingPriceComponent
+ acctOrderComp component.AccountingOrderComponent
+ notify *Notify
+ dlq *FeeDlq
+ notifyTimeOut *time.Timer
+ dlqTimeout *time.Timer
+}
+
+func NewCharging(natHandler mq.MessageQueue, config *config.Config) *Charging {
+ charge := &Charging{
+ sysMQ: natHandler,
+ acctSMComp: component.NewAccountingStatementComponent(),
+ acctUserComp: component.NewAccountingUserComponent(),
+ acctEvtComp: component.NewAccountingEventComponent(),
+ acctPriceComp: component.NewAccountingPriceComponent(),
+ acctOrderComp: component.NewAccountingOrderComponent(),
+ notify: NewNotify(natHandler),
+ dlq: NewFeeDlq(natHandler),
+ notifyTimeOut: time.NewTimer(idleDuration),
+ dlqTimeout: time.NewTimer(idleDuration),
+ }
+ return charge
+}
+
+func (c *Charging) Run() {
+ go c.startOrdering()
+ go c.startCharging()
+ go c.notify.Run()
+ go c.dlq.Run()
+}
+
+func (c *Charging) startOrdering() {
+ c.preOrder()
+}
+
+func (c *Charging) preOrder() {
+ var err error
+ var i int = 0
+ for {
+ i++
+ err = c.sysMQ.BuildOrderEventStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build order stream for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ break
+ }
+}
+
+func (c *Charging) startCharging() {
+ for {
+ c.preReadMsgs()
+ c.handleReadMsgs(10)
+ time.Sleep(2 * idleDuration)
+ }
+}
+
+func (c *Charging) preReadMsgs() {
+ var err error
+ var i int = 0
+ for {
+ i++
+ err = c.sysMQ.BuildFeeEventStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build event stream for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ break
+ }
+}
+
+func (c *Charging) handleReadMsgs(failedLimit int) {
+ failReadTime := 0
+ for {
+ if failReadTime >= failedLimit {
+ break
+ }
+ err := c.sysMQ.VerifyFeeEventStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to verify stream for the %d time", (failReadTime + 1))
+ slog.Error(tip, slog.Any("error", err))
+ failReadTime++
+ continue
+ }
+ msgs, err := c.sysMQ.FetchFeeEventMessages(5)
+ if err == nats.ErrTimeout {
+ continue
+ }
+
+ if err != nil {
+ tip := fmt.Sprintf("fail to fetch event messages for the %d time", (failReadTime + 1))
+ slog.Error(tip, slog.Any("error", err))
+ failReadTime++
+ continue
+ }
+ if msgs == nil {
+ tip := fmt.Sprintf("msgs is null for the %d time", (failReadTime + 1))
+ slog.Warn(tip)
+ failReadTime++
+ continue
+ }
+
+ for msg := range msgs.Messages() {
+ c.handleMsgWithRetry(msg)
+ }
+
+ }
+}
+
+func (c *Charging) handleMsgWithRetry(msg jetstream.Msg) {
+ strData := string(msg.Data())
+ slog.Debug("Fee->received", slog.Any("msg.subject", msg.Subject()), slog.Any("msg.data", strData))
+ // A maximum of 3 attempts
+ var err error = nil
+ for j := 0; j < 3; j++ {
+ err = c.handleMsgData(msg)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ slog.Error("accounting handles a single msg with 3 retries", slog.Any("msg.data", strData), slog.Any("error", err))
+ // move DLQ for fail to handle message
+ err = c.moveMsgToDLQWithReTry(msg, 5)
+ if err != nil {
+ tip := fmt.Sprintf("failed to move fee msg to DLQ with %d retries", 5)
+ slog.Error(tip, slog.Any("msg.data", strData), slog.Any("error", err))
+ }
+ }
+
+ // The code for acknowledging the processing of a single metering message is done.
+ err = msg.Ack()
+ if err != nil {
+ slog.Warn("failed to ack after processing fee msg", slog.Any("msg.data", strData), slog.Any("error", err))
+ }
+}
+
+func (c *Charging) moveMsgToDLQWithReTry(msg jetstream.Msg, limit int) error {
+ // A maximum of five attempts for move DLQ
+ var err error = nil
+ for i := 0; i < limit; i++ {
+ err = c.moveMsgToDLQ(msg)
+ if err == nil {
+ break
+ }
+ }
+ return err
+}
+
+func (c *Charging) moveMsgToDLQ(msg jetstream.Msg) error {
+ c.dlqTimeout.Reset(idleDuration)
+ select {
+ case c.dlq.CH <- msg.Data():
+ return nil
+ case <-c.dlqTimeout.C:
+ return fmt.Errorf("try to move fee DLQ with timeout, %v", idleDuration)
+ }
+}
+
+func (c *Charging) handleMsgData(msg jetstream.Msg) error {
+ event, extraMap, err := c.parseMessage(msg)
+ if err != nil {
+ return err
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ err = c.logAndVerifyEvent(ctx, event)
+ if err != nil {
+ if errors.Is(err, types.ErrDuplicatedEvent) {
+ slog.Warn("duplicated accounting event uuid", slog.Any("event", event))
+ return nil
+ } else {
+ return err
+ }
+ }
+
+ ap, err := c.verifyAndGetPriceOfFee(ctx, event, extraMap)
+ if err != nil {
+ slog.Warn("did not find a valid price", slog.Any("event", event), slog.Any("extraMap", extraMap), slog.Any("error", err))
+ return nil
+ }
+
+ err = c.acctUserComp.CheckAccountingUser(ctx, event.UserUUID)
+ if err != nil {
+ return fmt.Errorf("fail to check user balance, %v, %w", event.UserUUID, err)
+ }
+ slog.Debug("use sku price", slog.Any("price", ap), slog.Any("event", event))
+
+ err = c.processingFee(ctx, event, ap)
+ if err != nil {
+ return fmt.Errorf("fail to add statement and change balance, %v, %w", event, err)
+ }
+ c.checkBalanceAndSendNotification(ctx, event)
+ return nil
+}
+
+func (c *Charging) logAndVerifyEvent(ctx context.Context, event *types.AcctEvent) error {
+ err := c.acctEvtComp.AddNewAccountingEvent(ctx, event)
+ if err != nil {
+ return fmt.Errorf("fail to record fee event log, %v, %w", event, err)
+ }
+ st, err := c.acctSMComp.FindStatementByEventID(ctx, event)
+ if err != nil {
+ return fmt.Errorf("fail to check fee event statement, %v, %w", event, err)
+ }
+ if st != nil {
+ return types.ErrDuplicatedEvent
+ }
+ return nil
+}
+
+func (c *Charging) verifyAndGetPriceOfFee(ctx context.Context, event *types.AcctEvent, extraMap map[string]string) (*database.AccountPrice, error) {
+ if extraMap != nil {
+ detailIDStr, ok := extraMap[types.OrderDetailID]
+ if ok && detailIDStr != "" {
+ detailIDInt, err := strconv.ParseInt(detailIDStr, 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("convert order detail id %s to int64, %w", detailIDStr, err)
+ }
+ detail, err := c.acctOrderComp.GetDetailByID(ctx, detailIDInt)
+ if err != nil {
+ return nil, fmt.Errorf("did not find order detail by id %d, %w", detailIDInt, err)
+ }
+ // check if order is expired
+ if event.CreatedAt.After(detail.BeginTime) && event.CreatedAt.Before(detail.EndTime) {
+ ap, err := c.acctPriceComp.GetPriceByID(ctx, detail.SkuPriceID)
+ if err != nil {
+ return nil, fmt.Errorf("query price by id, %d, %w", detail.SkuPriceID, err)
+ }
+ return ap, nil
+ } else {
+ // send order expired notification
+ c.sendOrderExpiredNotificationWithRetry(event, detail)
+ }
+ }
+ }
+
+ priceReq := types.AcctPriceQueryReq{
+ SkuType: utils.GetSKUTypeByScene(types.SceneType(event.Scene)),
+ ResourceID: event.ResourceID,
+ SkuKind: types.SKUPayAsYouGo,
+ SkuUnitType: utils.GetSkuUnitTypeByScene(types.SceneType(event.Scene)),
+ PriceTime: event.CreatedAt,
+ }
+ ap, err := c.acctPriceComp.GetLatestByTime(ctx, priceReq)
+ if err != nil {
+ return nil, fmt.Errorf("query latest price by time, %v, %w", priceReq, err)
+ }
+ return ap, nil
+}
+
+func (c *Charging) processingFee(ctx context.Context, event *types.AcctEvent, ap *database.AccountPrice) error {
+ cost := 0.0
+ if ap.SkuKind == types.SKUPayAsYouGo {
+ cost = (0 - float64(ap.SkuPrice)*float64(event.Value)/float64(ap.SkuUnit))
+ }
+ eventReq := &types.ACCT_EVENT_REQ{
+ EventUUID: event.Uuid,
+ UserUUID: event.UserUUID,
+ Value: cost,
+ Scene: types.SceneType(event.Scene),
+ OpUID: event.OpUID,
+ CustomerID: event.CustomerID,
+ EventDate: event.CreatedAt,
+ Price: float64(ap.SkuPrice),
+ Consumption: float64(event.Value),
+ ValueType: event.ValueType,
+ ResourceID: event.ResourceID,
+ ResourceName: event.ResourceName,
+ SkuID: ap.ID,
+ RecordedAt: event.CreatedAt,
+ SkuUnit: ap.SkuUnit,
+ SkuUnitType: ap.SkuUnitType,
+ SkuPriceCurrency: ap.SkuPriceCurrency,
+ }
+ err := c.acctSMComp.AddNewStatement(ctx, eventReq)
+ if err != nil {
+ return fmt.Errorf("fail to add statement and change balance, %v, %w", eventReq, err)
+ }
+ return nil
+}
+
+func (c *Charging) parseMessage(msg jetstream.Msg) (*types.AcctEvent, map[string]string, error) {
+ event, err := c.parseMessageData(msg)
+ if err != nil {
+ return nil, nil, fmt.Errorf("parse msg data, %w", err)
+ }
+ extraMap, err := c.parseMessageExtraData(event.Extra)
+ if err != nil {
+ return nil, nil, fmt.Errorf("parse msg extra data, %w", err)
+ }
+ return event, extraMap, nil
+}
+
+func (c *Charging) parseMessageData(msg jetstream.Msg) (*types.AcctEvent, error) {
+ strData := string(msg.Data())
+ evt := types.AcctEvent{}
+ err := json.Unmarshal(msg.Data(), &evt)
+ if err != nil {
+ return nil, fmt.Errorf("fail to unmarshal fee event, %v, %w", strData, err)
+ }
+ return &evt, nil
+}
+
+func (c *Charging) parseMessageExtraData(extra string) (map[string]string, error) {
+ extraMap := make(map[string]string, 0)
+ if len(strings.Trim(extra, " ")) == 0 {
+ return extraMap, nil
+ }
+ err := json.Unmarshal([]byte(extra), &extraMap)
+ if err != nil {
+ return nil, fmt.Errorf("fail to unmarshal fee extra json data, %v, %w", extra, err)
+ }
+ return extraMap, nil
+}
+
+func (c *Charging) checkBalanceAndSendNotification(ctx context.Context, event *types.AcctEvent) {
+ account, err := c.acctUserComp.GetAccountingByUserID(ctx, event.UserUUID)
+ if err != nil {
+ slog.Warn("fail to query account before check account balance", slog.Any("user_uuid", event.UserUUID), slog.Any("error", err))
+ return
+ }
+ if account == nil {
+ return
+ }
+ if account.Balance <= 0 {
+ // retry 3 times for notification
+ for i := 0; i < 3; i++ {
+ err = c.sendNotification(int(types.ACCTLackBalance), "insufficient funds", event)
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ slog.Error("fail to notify for retry 3 times", slog.Any("event", event), slog.Any("error", err))
+ }
+ }
+}
+
+func (c *Charging) sendNotification(reasonCode int, reasonMsg string, event *types.AcctEvent) error {
+ notify := types.AcctNotify{
+ CreatedAt: time.Now(),
+ ReasonCode: reasonCode,
+ ReasonMsg: reasonMsg,
+ }
+ if event != nil {
+ notify.Uuid = event.Uuid
+ notify.UserUUID = event.UserUUID
+ }
+ c.notifyTimeOut.Reset(idleDuration)
+ select {
+ case c.notify.CH <- notify:
+ return nil
+ case <-c.notifyTimeOut.C:
+ return fmt.Errorf("try to sent notification with timeout, %v", idleDuration)
+ }
+}
+
+func (c *Charging) sendOrderExpiredNotificationWithRetry(event *types.AcctEvent, orderDetail *database.AccountOrderDetail) {
+ orderExpiredNotify := types.AcctOrderExpiredEvent{
+ OrderUUID: orderDetail.OrderUUID,
+ UserUUID: event.UserUUID,
+ DetailID: orderDetail.ID,
+ ResourceID: orderDetail.ResourceID,
+ BeginTime: orderDetail.BeginTime,
+ EndTime: orderDetail.EndTime,
+ CreatedAt: time.Now(),
+ }
+ var err error
+ for m := 0; m < 5; m++ {
+ err = c.sendOrderExpiredNotification(orderExpiredNotify)
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ slog.Error("fail to pub order expired notification with retry 5 times", slog.Any("notify", orderExpiredNotify), slog.Any("error", err))
+ }
+}
+
+func (c *Charging) sendOrderExpiredNotification(orderExpiredNotify types.AcctOrderExpiredEvent) error {
+ str, err := json.Marshal(orderExpiredNotify)
+ if err != nil {
+ return fmt.Errorf("fail to marshal order expired notification, %v, %w", orderExpiredNotify, err)
+ }
+ err = c.sysMQ.PublishOrderExpiredData(str)
+ if err != nil {
+ return fmt.Errorf("fail to pub order expired notification, %v, %w", string(str), err)
+ }
+ return nil
+}
+
+
+
package consumer
+
+import (
+ "fmt"
+ "log/slog"
+ "time"
+
+ "opencsg.com/csghub-server/mq"
+)
+
+type FeeDlq struct {
+ sysMQ mq.MessageQueue
+ CH chan []byte
+ timeOut *time.Timer
+}
+
+func NewFeeDlq(mqh mq.MessageQueue) *FeeDlq {
+ dlq := &FeeDlq{
+ sysMQ: mqh,
+ CH: make(chan []byte),
+ timeOut: time.NewTimer(idleDuration),
+ }
+ return dlq
+}
+
+func (d *FeeDlq) Run() {
+ for {
+ d.preDLQ()
+ d.moveWithRetry()
+ }
+}
+
+func (d *FeeDlq) preDLQ() {
+ var err error
+ var i int = 0
+ for {
+ i++
+ err = d.sysMQ.BuildDLQStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build DLQ stream in accounting for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ break
+ }
+}
+
+func (d *FeeDlq) moveWithRetry() {
+ for {
+ err := d.sysMQ.VerifyDLQStream()
+ if err != nil {
+ slog.Error("fail to verify DLQ stream", slog.Any("error", err))
+ break
+ }
+
+ var data []byte = []byte("")
+ d.timeOut.Reset(idleDuration)
+ select {
+ case data = <-d.CH:
+ case <-d.timeOut.C:
+ }
+
+ if len(data) < 1 {
+ continue
+ }
+ err = d.publishToDLQWithRetry(data)
+ if err != nil {
+ break
+ }
+ }
+}
+
+func (d *FeeDlq) publishToDLQWithRetry(data []byte) error {
+ // max try 10 times
+ var err error = nil
+ for m := 0; m < 10; m++ {
+ err = d.sysMQ.PublishFeeDataToDLQ(data)
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ slog.Error("fail to move DLQ with retry 10 times", slog.Any("data", string(data)), slog.Any("error", err))
+ }
+ return err
+}
+
+
+
package consumer
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/nats-io/nats.go"
+ "github.com/nats-io/nats.go/jetstream"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mq"
+)
+
+type Metering struct {
+ sysMQ mq.MessageQueue
+ meterComp component.MeteringComponent
+ chargingEnable bool
+}
+
+func NewMetering(natHandler mq.MessageQueue, config *config.Config) *Metering {
+ meter := &Metering{
+ sysMQ: natHandler,
+ meterComp: component.NewMeteringComponent(),
+ chargingEnable: config.Accounting.ChargingEnable,
+ }
+ return meter
+}
+
+func (m *Metering) Run() {
+ go m.startMetering()
+}
+
+func (m *Metering) startMetering() {
+ for {
+ m.preReadMsgs()
+ m.handleReadMsgs(10)
+ time.Sleep(2 * idleDuration)
+ }
+}
+
+func (m *Metering) preReadMsgs() {
+ var err error
+ var i int = 0
+ for {
+ i++
+ err = m.sysMQ.BuildMeterEventStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build metering stream for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ err = m.sysMQ.BuildDLQStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build DLQ stream in metering for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ break
+ }
+}
+
+func (m *Metering) handleReadMsgs(failedLimit int) {
+ failReadTime := 0
+ for {
+ if failReadTime >= failedLimit {
+ break
+ }
+ err := m.sysMQ.VerifyMeteringStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to verify metering stream for the %d time", (failReadTime + 1))
+ slog.Error(tip, slog.Any("err", err))
+ failReadTime++
+ continue
+ }
+ msgs, err := m.sysMQ.FetchMeterEventMessages(5)
+ if err == nats.ErrTimeout {
+ continue
+ }
+
+ if err != nil {
+ tip := fmt.Sprintf("fail to fetch metering messages for the %d time", (failReadTime + 1))
+ slog.Error(tip, slog.Any("err", err))
+ failReadTime++
+ continue
+ }
+ if msgs == nil {
+ tip := fmt.Sprintf("metering msgs is null for the %d time", (failReadTime + 1))
+ slog.Warn(tip)
+ failReadTime++
+ continue
+ }
+
+ for msg := range msgs.Messages() {
+ m.handleMsgWithRetry(msg)
+ }
+ }
+}
+
+func (m *Metering) handleMsgWithRetry(msg jetstream.Msg) {
+ strData := string(msg.Data())
+ slog.Debug("Meter->received", slog.Any("msg.subject", msg.Subject()), slog.Any("msg.data", strData))
+ // A maximum of 3 attempts
+ var (
+ err error = nil
+ evt *types.METERING_EVENT = nil
+ )
+ for j := 0; j < 3; j++ {
+ evt, err = m.handleMsgData(msg)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ slog.Error("metering handles a single msg with 3 retries", slog.Any("msg.data", strData), slog.Any("error", err))
+ // move to DLQ for failed to handle message
+ err = m.moveMsgToDLQWithReTry(msg, 5)
+ if err != nil {
+ tip := fmt.Sprintf("failed to move meter msg to DLQ with %d retries", 5)
+ slog.Error(tip, slog.Any("msg.data", string(msg.Data())), slog.Any("error", err))
+ }
+ } else {
+ if m.chargingEnable {
+ err = m.pubFeeEventWithReTry(msg, evt, 5)
+ if err != nil {
+ tip := fmt.Sprintf("failed to pub fee event msg with %d retries", 5)
+ slog.Error(tip, slog.Any("msg.data", string(msg.Data())), slog.Any("error", err))
+ // todo: need more discuss on how to persist failed message finally
+ }
+ }
+ }
+
+ // ack for handle metering message done
+ err = msg.Ack()
+ if err != nil {
+ slog.Warn("failed to ack after processing meter msg", slog.Any("msg.data", strData), slog.Any("error", err))
+ }
+}
+
+func (m *Metering) handleMsgData(msg jetstream.Msg) (*types.METERING_EVENT, error) {
+ event, err := m.parseMessageData(msg)
+ if err != nil {
+ return nil, err
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err = m.meterComp.SaveMeteringEventRecord(ctx, event)
+ if err != nil {
+ return nil, fmt.Errorf("failed to save meter event, %v, %w", event, err)
+ }
+ return event, nil
+}
+
+func (m *Metering) parseMessageData(msg jetstream.Msg) (*types.METERING_EVENT, error) {
+ strData := string(msg.Data())
+ evt := types.METERING_EVENT{}
+ err := json.Unmarshal(msg.Data(), &evt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to unmarshal meter event, %v, %w", strData, err)
+ }
+ return &evt, nil
+}
+
+func (m *Metering) pubFeeEventWithReTry(msg jetstream.Msg, evt *types.METERING_EVENT, limit int) error {
+ // A maximum of five attempts for pub fee event
+ var err error
+ for i := 0; i < limit; i++ {
+ switch evt.ValueType {
+ case types.TimeDurationMinType:
+ err = m.sysMQ.PublishFeeCreditData(msg.Data())
+ case types.TokenNumberType:
+ err = m.sysMQ.PublishFeeTokenData(msg.Data())
+ case types.QuotaNumberType:
+ err = m.sysMQ.PublishFeeQuotaData(msg.Data())
+ }
+ if err == nil {
+ break
+ }
+ }
+ return err
+}
+
+func (m *Metering) moveMsgToDLQWithReTry(msg jetstream.Msg, limit int) error {
+ // A maximum of five attempts for move DLQ
+ var err error
+ for i := 0; i < limit; i++ {
+ err = m.sysMQ.PublishMeterDataToDLQ(msg.Data())
+ if err == nil {
+ break
+ }
+ }
+ return err
+}
+
+
+
package consumer
+
+import (
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mq"
+)
+
+type Notify struct {
+ sysMQ mq.MessageQueue
+ CH chan types.AcctNotify
+ timeOut *time.Timer
+}
+
+func NewNotify(mqh mq.MessageQueue) *Notify {
+ notify := &Notify{
+ sysMQ: mqh,
+ CH: make(chan types.AcctNotify),
+ timeOut: time.NewTimer(idleDuration),
+ }
+ return notify
+}
+
+func (n *Notify) Run() {
+ for {
+ n.preNotify()
+ n.notifyWithRetry()
+ }
+}
+
+func (n *Notify) preNotify() {
+ var err error
+ var i int = 0
+ for {
+ i++
+ err = n.sysMQ.BuildNotifyStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build notify stream for the %d time", i)
+ slog.Error(tip, slog.Any("err", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ break
+ }
+}
+
+func (n *Notify) notifyWithRetry() {
+ for {
+ err := n.sysMQ.VerifyNotifyStream()
+ if err != nil {
+ slog.Error("fail to verify notify stream", slog.Any("error", err))
+ break
+ }
+
+ var notify types.AcctNotify = types.AcctNotify{ReasonCode: -1}
+ n.timeOut.Reset(idleDuration)
+ select {
+ case notify = <-n.CH:
+ case <-n.timeOut.C:
+ }
+
+ if notify.ReasonCode == -1 {
+ continue
+ }
+ err = n.publishNotificationWithRetry(notify)
+ if err != nil {
+ break
+ }
+ }
+}
+
+func (n *Notify) publishNotificationWithRetry(notify types.AcctNotify) error {
+ // max try 5 times
+ var err error
+ for m := 0; m < 5; m++ {
+ err = n.publishNotification(notify)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ slog.Error("fail to pub notify with retry 5 times", slog.Any("notify", notify), slog.Any("error", err))
+ }
+ return err
+}
+
+func (n *Notify) publishNotification(notify types.AcctNotify) error {
+ str, err := json.Marshal(notify)
+ if err != nil {
+ return fmt.Errorf("fail to marshal for notification of lack balance, %v, %w", notify, err)
+ }
+ err = n.sysMQ.PublishNotificationForNoBalance(str)
+ if err != nil {
+ return fmt.Errorf("fail to publish notification of lack balance, %v, %w", notify, err)
+ }
+ return nil
+}
+
+
+
package consumer
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/nats-io/nats.go"
+ "github.com/nats-io/nats.go/jetstream"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mq"
+)
+
+type RechargeConsumer struct {
+ sysMQ mq.MessageQueue
+ rechargeComp component.RechargeComponent
+ accountingStatementComp component.AccountingStatementComponent
+}
+
+func NewRechargeConsumer(natHandler mq.MessageQueue, config *config.Config) *RechargeConsumer {
+ rechargeConsumer := &RechargeConsumer{
+ sysMQ: natHandler,
+ rechargeComp: component.NewRechargeComponent(config),
+ accountingStatementComp: component.NewAccountingStatementComponent(),
+ }
+ return rechargeConsumer
+}
+
+func (m *RechargeConsumer) Run() {
+ go m.start()
+}
+
+func (m *RechargeConsumer) start() {
+ for {
+ m.preReadMsgs()
+ m.handleReadMsgs(10)
+ time.Sleep(2 * idleDuration)
+ }
+}
+
+func (m *RechargeConsumer) preReadMsgs() {
+ var err error
+ var i = 0
+ for {
+ i++
+ err = m.sysMQ.BuildRechargeEventStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build recharge stream for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ err = m.sysMQ.BuildDLQStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to build DLQ stream in recharge for the %d time", i)
+ slog.Error(tip, slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ break
+ }
+}
+
+func (m *RechargeConsumer) handleReadMsgs(failedLimit int) {
+ failReadTime := 0
+ for {
+ if failReadTime >= failedLimit {
+ break
+ }
+ err := m.sysMQ.VerifyRechargeStream()
+ if err != nil {
+ tip := fmt.Sprintf("fail to verify recharge stream for the %d time", (failReadTime + 1))
+ slog.Error(tip, slog.Any("err", err))
+ failReadTime++
+ continue
+ }
+ msgs, err := m.sysMQ.FetchRechargeEventMessages(5)
+ if err == nats.ErrTimeout {
+ continue
+ }
+
+ if err != nil {
+ tip := fmt.Sprintf("fail to fetch recharge messages for the %d time", (failReadTime + 1))
+ slog.Error(tip, slog.Any("err", err))
+ failReadTime++
+ continue
+ }
+ if msgs == nil {
+ tip := fmt.Sprintf("recharge msgs is null for the %d time", (failReadTime + 1))
+ slog.Warn(tip)
+ failReadTime++
+ continue
+ }
+
+ for msg := range msgs.Messages() {
+ m.handleMsgWithRetry(msg)
+ }
+ }
+}
+
+func (m *RechargeConsumer) handleMsgWithRetry(msg jetstream.Msg) {
+ strData := string(msg.Data())
+ slog.Debug("Recharge->received",
+ slog.Any("msg.subject", msg.Subject()),
+ slog.Any("msg.data", strData))
+ // A maximum of 3 attempts
+ var (
+ err error = nil
+ )
+ for j := 0; j < 3; j++ {
+ _, err = m.handleMsgData(msg)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ slog.Error("recharge handles a single msg with 3 retries",
+ slog.Any("msg.data", strData),
+ slog.Any("error", err))
+ // move to DLQ for failed to handle message
+ err = m.moveMsgToDLQWithReTry(msg, 5)
+ if err != nil {
+ tip := fmt.Sprintf("failed to move recharge msg to DLQ with %d retries", 5)
+ slog.Error(tip, slog.Any("msg.data", string(msg.Data())), slog.Any("error", err))
+ }
+ }
+ // ack for handle recharge message done
+ err = msg.Ack()
+ if err != nil {
+ slog.Warn("failed to ack after processing consumer msg",
+ slog.Any("msg.data", strData),
+ slog.Any("error", err))
+ }
+}
+
+func (m *RechargeConsumer) handleMsgData(msg jetstream.Msg) (*types.PaymentNotifyEvent, error) {
+ event, err := m.parseMessageData(msg)
+ if err != nil {
+ return nil, err
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Second)
+ defer cancel()
+
+ if event.Extra == types.RechargeExtraType {
+ orderNo := event.OrderNo
+ if event.Paid {
+ recharge, err := m.rechargeComp.RechargeSucceed(ctx, orderNo)
+ if err != nil {
+ return nil, err
+ }
+ req := types.RECHARGE_REQ{
+ Scene: types.SceneCashCharge,
+ OpUID: recharge.FromUserUUID,
+ Value: float64(recharge.Amount),
+ }
+ rechargeUUID, err := uuid.Parse(recharge.RechargeUUID)
+ if err != nil {
+ return nil, err
+ }
+ err = m.accountingStatementComp.RechargeAccountingUser(ctx, recharge.UserUUID, req, rechargeUUID)
+ if err != nil {
+ return nil, err
+ }
+ } else if event.Reversed {
+ _, err := m.rechargeComp.RechargeClosed(ctx, orderNo)
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+ return event, nil
+}
+
+func (m *RechargeConsumer) parseMessageData(msg jetstream.Msg) (*types.PaymentNotifyEvent, error) {
+ strData := string(msg.Data())
+ evt := types.PaymentNotifyEvent{}
+ err := json.Unmarshal(msg.Data(), &evt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to unmarshal recharge event, %v, %w", strData, err)
+ }
+ return &evt, nil
+}
+
+func (m *RechargeConsumer) moveMsgToDLQWithReTry(msg jetstream.Msg, limit int) error {
+ // A maximum of five attempts for move DLQ
+ var err error
+ for i := 0; i < limit; i++ {
+ err = m.sysMQ.PublishRechargeDataToDLQ(msg.Data())
+ if err == nil {
+ break
+ }
+ }
+ return err
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/accounting/utils"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func NewCreditHandler(config *config.Config) (*CreditHandler, error) {
+ return &CreditHandler{
+ auc: component.NewAccountingUserComponent(),
+ asc: component.NewAccountingStatementComponent(),
+ abc: component.NewAccountingBillComponent(),
+ apc: component.NewAccountingPresentComponent(),
+ }, nil
+}
+
+type CreditHandler struct {
+ auc component.AccountingUserComponent
+ asc component.AccountingStatementComponent
+ abc component.AccountingBillComponent
+ apc component.AccountingPresentComponent
+}
+
+func (ch *CreditHandler) QueryAllUsersBalance(ctx *gin.Context) {
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ accounts, total, err := ch.auc.ListAccountingUser(ctx, per, page)
+ if err != nil {
+ slog.Error("fail to list all accounts", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": accounts,
+ "total": total,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+func (ch *CreditHandler) QueryBalanceByUserID(ctx *gin.Context) {
+ userUUID := ctx.Param("id")
+ if len(userUUID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ account, err := ch.auc.GetAccountingByUserID(ctx, userUUID)
+ if err != nil || account == nil {
+ slog.Error("fail to query account by user id", slog.Any("userid", userUUID), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, account)
+}
+
+func (ch *CreditHandler) QueryStatementByUserID(ctx *gin.Context) {
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ scene, err := utils.GetSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userID := ctx.Param("id")
+ instance_name := ctx.Query("instance_name")
+ startTime := ctx.Query("start_time") // format: '2024-06-12 08:27:22'
+ endTime := ctx.Query("end_time") // format: '2024-06-12 17:17:22'
+ if len(startTime) < 1 || len(endTime) < 1 || len(userID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ if !utils.ValidateDateTimeFormat(startTime, "2006-01-02 15:04:05") || !utils.ValidateDateTimeFormat(endTime, "2006-01-02 15:04:05") {
+ slog.Error("Bad request datetime format")
+ httpbase.BadRequest(ctx, "Bad request datetime format")
+ return
+ }
+
+ req := types.ACCT_STATEMENTS_REQ{
+ UserUUID: userID,
+ Scene: scene,
+ InstanceName: instance_name,
+ StartTime: startTime,
+ EndTime: endTime,
+ Per: per,
+ Page: page,
+ }
+
+ respData, err := ch.asc.ListStatementByUserIDAndTime(ctx, req)
+ if err != nil {
+ slog.Error("fail to query statement by user", slog.Any("userid", userID), slog.Any("start_time", startTime), slog.Any("end_time", endTime), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, respData)
+}
+
+func (ch *CreditHandler) QueryBillsByUserID(ctx *gin.Context) {
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ scene, err := utils.GetSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userUUID := ctx.Param("id")
+ startDate := ctx.Query("start_date") // format: '2024-06-12'
+ endDate := ctx.Query("end_date") // format: '2024-06-12'
+ if len(startDate) < 1 || len(endDate) < 1 || len(userUUID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+
+ if !utils.ValidateDateTimeFormat(startDate, "2006-01-02") || !utils.ValidateDateTimeFormat(endDate, "2006-01-02") {
+ slog.Error("Bad request date format")
+ httpbase.BadRequest(ctx, "Bad request date format")
+ return
+ }
+
+ req := types.ACCT_BILLS_REQ{
+ UserUUID: userUUID,
+ Scene: scene,
+ StartDate: startDate,
+ EndDate: endDate,
+ Per: per,
+ Page: page,
+ }
+
+ respData, err := ch.abc.ListBillsByUserIDAndDate(ctx, req)
+ if err != nil {
+ slog.Error("fail to query bills by user", slog.Any("userUUID", userUUID), slog.Any("start_date", startDate), slog.Any("end_date", endDate), slog.Any("scene", scene), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, respData)
+}
+
+func (ch *CreditHandler) RechargeByUserID(ctx *gin.Context) {
+ var req types.RECHARGE_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad recharge request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ if req.Value < 0 {
+ slog.Error("Bad recharge value")
+ httpbase.BadRequest(ctx, "Bad recharge value")
+ return
+ }
+ userUUID := ctx.Param("id")
+ if len(userUUID) < 1 {
+ slog.Error("Bad recharge user id")
+ httpbase.BadRequest(ctx, "Bad recharge user id")
+ return
+ }
+ err = ch.auc.CheckAccountingUser(ctx, userUUID)
+ if err != nil {
+ slog.Error("fail to check user balance", slog.Any("user_uuid", userUUID), slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = ch.asc.RechargeAccountingUser(ctx, userUUID, req, uuid.New())
+ if err != nil {
+ slog.Error("fail to recharge account by user", slog.Any("user_uuid", userUUID), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ account, err := ch.auc.GetAccountingByUserID(ctx, userUUID)
+ if err != nil || account == nil {
+ slog.Error("fail to get account by user id", slog.Any("user_uuid", userUUID), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, account)
+}
+
+func (ch *CreditHandler) SendPresentByUserID(ctx *gin.Context) {
+ var req types.ACTIVITY_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad present request format", slog.Any("req", req), slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userUUID := ctx.Param("id")
+ if len(userUUID) < 1 {
+ slog.Error("Bad present user uuid", slog.Any("req", req))
+ httpbase.BadRequest(ctx, "Bad present user uuid")
+ return
+ }
+ err = ch.auc.CheckAccountingUser(ctx, userUUID)
+ if err != nil {
+ slog.Error("fail to check user balance", slog.Any("user_uuid", userUUID), slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = ch.apc.PresentAccountingUser(ctx, userUUID, req)
+ if err != nil {
+ slog.Error("fail to send present by user", slog.Any("user_uuid", userUUID), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/accounting/utils"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func NewMeteringHandler() (*MeteringHandler, error) {
+ return &MeteringHandler{
+ amc: component.NewMeteringComponent(),
+ }, nil
+}
+
+type MeteringHandler struct {
+ amc component.MeteringComponent
+}
+
+func (mh *MeteringHandler) QueryMeteringStatementByUserID(ctx *gin.Context) {
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request pagination format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ scene, err := utils.GetSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userID := ctx.Param("id")
+ instance_name := ctx.Query("instance_name")
+ startTime := ctx.Query("start_time") // format: '2024-06-12 08:27:22'
+ endTime := ctx.Query("end_time") // format: '2024-06-12 17:17:22'
+ if len(startTime) < 1 || len(endTime) < 1 || len(userID) < 1 {
+ slog.Error("Bad request parameters format")
+ httpbase.BadRequest(ctx, "Bad request parameters format")
+ return
+ }
+ if !utils.ValidateDateTimeFormat(startTime, "2006-01-02 15:04:05") || !utils.ValidateDateTimeFormat(endTime, "2006-01-02 15:04:05") {
+ slog.Error("Bad request datetime format")
+ httpbase.BadRequest(ctx, "Bad request datetime format")
+ return
+ }
+
+ req := types.ACCT_STATEMENTS_REQ{
+ UserUUID: userID,
+ Scene: scene,
+ InstanceName: instance_name,
+ StartTime: startTime,
+ EndTime: endTime,
+ Per: per,
+ Page: page,
+ }
+
+ meters, total, err := mh.amc.ListMeteringByUserIDAndDate(ctx, req)
+ if err != nil {
+ slog.Error("fail to query meters by user", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": meters,
+ "total": total,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+func (mh *MeteringHandler) QueryMeteringStatByDate(ctx *gin.Context) {
+ scene, err := utils.GetSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ startDate := ctx.Query("start_date") // format: '2024-06-12'
+ endDate := ctx.Query("end_date") // format: '2024-06-12'
+
+ if !utils.ValidateDateTimeFormat(startDate, "2006-01-02") || !utils.ValidateDateTimeFormat(endDate, "2006-01-02") {
+ slog.Error("Bad request date format")
+ httpbase.BadRequest(ctx, "Bad request date format")
+ return
+ }
+
+ req := types.ACCT_STATEMENTS_REQ{
+ Scene: scene,
+ StartTime: startDate + " 00:00:00",
+ EndTime: endDate + " 23:59:59",
+ }
+
+ res, err := mh.amc.GetMeteringStatByDate(ctx, req)
+ if err != nil {
+ slog.Error("fail to query meter stat", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, res)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var ErrUserNotFound = errors.New("user not found, please login first")
+
+func NewMultiSyncHandler() (*MultiSyncHandler, error) {
+ return &MultiSyncHandler{
+ asqc: component.NewAccountingQuotaComponent(),
+ asqsc: component.NewAccountingQuotaStatementComponent(),
+ }, nil
+}
+
+type MultiSyncHandler struct {
+ asqc component.AccountingSyncQuotaComponent
+ asqsc component.AccountingSyncQuotaStatementComponent
+}
+
+func (msh *MultiSyncHandler) CreateOrUpdateQuota(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, ErrUserNotFound)
+ return
+ }
+ var req types.ACCT_QUOTA_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad quota request format", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ quota, err := msh.asqc.CreateOrUpdateQuota(ctx, currentUser, req)
+ if err != nil {
+ slog.Error("fail to add or update account quota", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, quota)
+}
+
+func (msh *MultiSyncHandler) QueryQuota(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, ErrUserNotFound)
+ return
+ }
+
+ quota, err := msh.asqc.GetQuotaByID(ctx, currentUser)
+ if err != nil {
+ slog.Error("fail to get quota by user", slog.Any("currentUser", currentUser), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, quota)
+}
+
+func (msh *MultiSyncHandler) CreateQuotaStatement(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, ErrUserNotFound)
+ return
+ }
+ var req types.ACCT_QUOTA_STATEMENT_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad quota statement request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ quotaSM, err := msh.asqsc.CreateQuotaStatement(ctx, currentUser, req)
+ if err != nil {
+ slog.Error("fail to create quota statement by user", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, quotaSM)
+}
+
+func (msh *MultiSyncHandler) QueryQuotaStatement(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, ErrUserNotFound)
+ return
+ }
+
+ repoPath := ctx.Query("repo_path")
+ repoType := ctx.Query("repo_type")
+ req := types.ACCT_QUOTA_STATEMENT_REQ{
+ RepoPath: repoPath,
+ RepoType: repoType,
+ }
+
+ account, err := msh.asqsc.GetQuotaStatement(ctx, currentUser, req)
+ if err != nil {
+ slog.Error("fail to get account quota statement", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, account)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func NewOrderHandler() (*OrderHandler, error) {
+ return &OrderHandler{
+ nao: component.NewAccountingOrderComponent(),
+ }, nil
+}
+
+type OrderHandler struct {
+ nao component.AccountingOrderComponent
+}
+
+func (oh *OrderHandler) OrderCreate(ctx *gin.Context) {
+ var req types.AcctOrderCreateReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("bad order request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ order, err := oh.nao.Create(ctx, req, time.Now())
+ if err != nil {
+ slog.Error("fail to create order", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ order, err = oh.nao.GetByID(ctx, order.OrderUUID)
+ if err != nil {
+ slog.Error("fail to get created order", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, order)
+}
+
+func (oh *OrderHandler) OrderGetByID(ctx *gin.Context) {
+ orderUUID := ctx.Param("id")
+ if orderUUID == "" {
+ slog.Error("bad request uuid is empty")
+ httpbase.BadRequest(ctx, "bad request uuid is empty")
+ return
+ }
+
+ order, err := oh.nao.GetByID(ctx, orderUUID)
+ if err != nil {
+ slog.Error("fail to query order", slog.Any("orderUUID", orderUUID), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, order)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func NewPriceHandler() (*PriceHandler, error) {
+ return &PriceHandler{
+ apc: component.NewAccountingPriceComponent(),
+ }, nil
+}
+
+type PriceHandler struct {
+ apc component.AccountingPriceComponent
+}
+
+func (ph *PriceHandler) QueryPricesBySKUType(ctx *gin.Context) {
+ skyStr := ctx.Query("sku_type")
+ skuInt, err := strconv.Atoi(skyStr)
+ if err != nil {
+ slog.Error("Bad request sku type format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ skuKind := ctx.Query("sku_kind")
+ resID := ctx.Query("resource_id")
+ req := types.AcctPriceListReq{
+ SkuType: types.SKUType(skuInt),
+ SkuKind: skuKind,
+ ResourceID: resID,
+ Per: per,
+ Page: page,
+ }
+
+ prices, total, err := ph.apc.ListPricesBySKUType(ctx, req)
+
+ if err != nil {
+ slog.Error("fail to query prices by sku_type/sku_kind/resource_id", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": prices,
+ "total": total,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+func (ph *PriceHandler) GetPriceByID(ctx *gin.Context) {
+ idStr := ctx.Param("id")
+ if len(idStr) < 1 {
+ slog.Error("Bad request id format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ id, err := strconv.Atoi(idStr)
+ if err != nil {
+ slog.Error("Bad request id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ price, err := ph.apc.GetPriceByID(ctx, int64(id))
+ if err != nil {
+ slog.Error("fail to query price by id", slog.Any("id", id), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, price)
+}
+
+func (ph *PriceHandler) PriceCreate(ctx *gin.Context) {
+ var req types.AcctPriceCreateReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad price request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ price, err := ph.apc.CreatePrice(ctx, req)
+ if err != nil {
+ slog.Error("fail to add price", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, price)
+}
+
+func (ph *PriceHandler) PriceUpdate(ctx *gin.Context) {
+ idStr := ctx.Param("id")
+ if len(idStr) < 1 {
+ slog.Error("Bad request id format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ id, err := strconv.Atoi(idStr)
+ if err != nil {
+ slog.Error("Bad request id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req types.AcctPriceCreateReq
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad price request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ price, err := ph.apc.UpdatePrice(ctx, req, int64(id))
+ if err != nil {
+ slog.Error("fail to update price", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, price)
+}
+
+func (ph *PriceHandler) PriceDelete(ctx *gin.Context) {
+ idStr := ctx.Param("id")
+ if len(idStr) < 1 {
+ slog.Error("Bad request id format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ id, err := strconv.Atoi(idStr)
+ if err != nil {
+ slog.Error("Bad request id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = ph.apc.DeletePrice(ctx, int64(id))
+ if err != nil {
+ slog.Error("fail to delete price by id", slog.Any("id", id), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "github.com/gin-gonic/gin"
+ "log/slog"
+ "net/http"
+ "opencsg.com/csghub-server/accounting/component"
+ "opencsg.com/csghub-server/accounting/utils"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+)
+
+type RechargeHandler struct {
+ rechargeStore database.AccountRechargeStore
+ rechargeComponent component.RechargeComponent
+}
+
+func NewRechargeHandler(config *config.Config) (*RechargeHandler, error) {
+ handler := RechargeHandler{
+ rechargeStore: database.NewAccountRechargeStore(),
+ rechargeComponent: component.NewRechargeComponent(config),
+ }
+ return &handler, nil
+}
+
+// POST
+// CreateRecharge godoc
+// @Security ApiKey
+// @Summary Create recharge order
+// @Description Create recharge order
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param body body types.AcctRechargeReq{} true "Recharge request payload"
+// @Success 200 {object} types.AcctRechargeResp{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/recharge/create-pay-order [post]
+func (h *RechargeHandler) CreateRecharge(ctx *gin.Context) {
+ userUUID := httpbase.GetCurrentUserUUID(ctx)
+ var req types.AcctRechargeReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad recharge request format", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ rechargeComp := h.rechargeComponent
+ amount, err := money.NewMoneyFromYuan(req.RechargeAmount)
+ if err != nil {
+ slog.Error("Bad recharge request format for recharge amount", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ recharge, payment, err := rechargeComp.CreateRecharge(ctx, userUUID, userUUID, "OpenCSG社区充值", amount, req.ChannelCode)
+ if err != nil {
+ slog.Error("Bad recharge request format", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ acctRechargeResp := types.AcctRechargeResp{
+ Content: payment.CodeUrl,
+ RechargeUUID: recharge.RechargeUUID,
+ RechargeOrderNo: recharge.OrderNo,
+ Channel: req.ChannelCode,
+ CreateTime: recharge.CreatedAt,
+ }
+
+ httpbase.OK(ctx, acctRechargeResp)
+}
+
+// GET
+// CreateRecharge godoc
+// @Security ApiKey
+// @Summary Fetch recharge order status by recharge id
+// @Description Fetch recharge order status by recharge id
+// @Param id path string true "recharge uuid"
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.RechargeStatusResp{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/recharge/{id}/status [get]
+func (h *RechargeHandler) FetchRechargeStatus(ctx *gin.Context) {
+ rechargeUUID := ctx.Param("id")
+ recharge, err := h.rechargeStore.GetRecharge(ctx, rechargeUUID)
+ if err != nil {
+ slog.Error("fetch recharge fail", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ rechargeStatusResp := types.RechargeStatusResp{
+ RechargeUUID: rechargeUUID,
+ RechargeSucceeded: recharge.Succeeded,
+ }
+
+ httpbase.OK(ctx, rechargeStatusResp)
+}
+
+func (h *RechargeHandler) convertToRechargeResp(recharges []*database.AccountRecharge) []*types.RechargeResp {
+ rechargeResponses := make([]*types.RechargeResp, len(recharges))
+ for i, recharge := range recharges {
+ rechargeResponses[i] = &types.RechargeResp{
+ RechargeUUID: recharge.RechargeUUID,
+ OrderNo: recharge.OrderNo,
+ UserUUID: recharge.UserUUID,
+ FromUserUUID: recharge.FromUserUUID,
+ Amount: recharge.Amount,
+ Currency: recharge.Currency,
+ PaymentUUID: recharge.PaymentUUID,
+ PaymentChannel: string(recharge.Channel),
+ Succeeded: recharge.Succeeded,
+ Closed: recharge.Closed,
+ TimeSucceeded: recharge.TimeSucceeded,
+ CreatedAt: recharge.CreatedAt,
+ UpdatedAt: recharge.UpdatedAt,
+ Description: recharge.Description,
+ }
+ }
+ return rechargeResponses
+}
+
+// GET
+// QueryCurrentUserRechargeList godoc
+// @Security ApiKey
+// @Summary List current user recharge list by start_time and end_time and query
+// @Description List current user recharge list by start_time and end_time and query
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param recharge_status query string false "Recharge status, options: 'succeed, waitpay'"
+// @Param recharge_payment_type query string false "Recharge payment type, options: 'wx_pub_qr, alipay_qr'"
+// @Param query query string false "Query string against recharge uuid"
+// @Param start_time query string true "Start time, format: '2024-06-12 08:27:22'"
+// @Param end_time query string true "End time, format: '2024-06-12 17:17:22'"
+// @Param current_user query string true "Current user identifier"
+// @Param per query int false "Items per page" default(50)
+// @Param page query int false "Page number" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/recharge/user-recharge-list [get]
+func (h *RechargeHandler) QueryCurrentUserRechargeList(ctx *gin.Context) {
+ userUUID := httpbase.GetCurrentUserUUID(ctx)
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ limit := per
+ offset := (page - 1) * per
+
+ rechargeStatus := ctx.Query("recharge_status")
+ rechargePaymentType := ctx.Query("recharge_payment_type")
+ orderNoPattern := ctx.Query("query")
+ startTimeStr := ctx.Query("start_time")
+ endTimeStr := ctx.Query("end_time")
+
+ if startTimeStr != "" {
+ if !utils.ValidateDateTimeFormat(startTimeStr, "2006-01-02 15:04:05") {
+ slog.Error("Bad request datetime format")
+ httpbase.BadRequest(ctx, "Bad request datetime format")
+ return
+ }
+ }
+
+ if endTimeStr != "" {
+ if !utils.ValidateDateTimeFormat(endTimeStr, "2006-01-02 15:04:05") {
+ slog.Error("Bad request datetime format")
+ httpbase.BadRequest(ctx, "Bad request datetime format")
+ return
+ }
+ }
+
+ filter := database.RechargeFilter{
+ OrderNoPattern: orderNoPattern,
+ StartDate: startTimeStr,
+ EndDate: endTimeStr,
+ Limit: limit,
+ Offset: offset,
+ }
+
+ if rechargeStatus != "" {
+ statusMapping := map[string]struct {
+ succeeded bool
+ closed bool
+ }{
+ "succeed": {true, false},
+ "waitpay": {false, false},
+ "closed": {false, true},
+ }
+
+ if status, ok := statusMapping[rechargeStatus]; ok {
+ filter.Succeeded = &status.succeeded
+ filter.Closed = &status.closed
+ } else {
+ errMsg := "Invalid recharge_status, options are 'succeed', 'waitpay', or 'closed'"
+ slog.Error(errMsg)
+ httpbase.BadRequest(ctx, errMsg)
+ return
+ }
+ }
+
+ if rechargePaymentType != "" {
+ var paymentChannel consts.PaymentChannel
+ switch rechargePaymentType {
+ case "wx_pub_qr":
+ paymentChannel = consts.ChannelWxPubQr
+ case "alipay_qr":
+ paymentChannel = consts.ChannelAlipayQr
+ default:
+ errMsg := "Invalid recharge_payment_type, options are 'wx_pub_qr' or 'alipay_qr'"
+ slog.Error(errMsg)
+ httpbase.BadRequest(ctx, errMsg)
+ return
+ }
+ filter.PaymentChannel = &paymentChannel
+ }
+
+ rechargeStore := h.rechargeStore
+ recharges, err := rechargeStore.ListRecharges(ctx, userUUID, filter)
+ if err != nil {
+ slog.Error("Failed to list recharges", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ rechargesResp := h.convertToRechargeResp(recharges)
+
+ totalCount, err := rechargeStore.CountRecharges(ctx, userUUID, filter)
+ if err != nil {
+ slog.Error("Failed to count recharges", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ ctx.PureJSON(http.StatusOK, gin.H{
+ "msg": "OK",
+ "data": rechargesResp,
+ "total": totalCount,
+ })
+
+}
+
+
+
package router
+
+import (
+ "fmt"
+
+ "github.com/gin-gonic/gin"
+ "github.com/gin-gonic/gin/binding"
+ "github.com/go-playground/validator/v10"
+ "opencsg.com/csghub-server/accounting/handler"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func NewAccountRouter(config *config.Config) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+ r.Use(middleware.Authenticator(config))
+
+ err := createCustomValidator()
+ if err != nil {
+ return nil, fmt.Errorf("error create validator:%w", err)
+ }
+
+ // Credit routes
+ creditHandler, err := handler.NewCreditHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating credit handler:%w", err)
+ }
+
+ // multisync
+ multiSyncHandler, err := handler.NewMultiSyncHandler()
+ if err != nil {
+ return nil, fmt.Errorf("error creating multi sync handler:%w", err)
+ }
+
+ // metering
+ meterHandler, err := handler.NewMeteringHandler()
+ if err != nil {
+ return nil, fmt.Errorf("error creating multi sync handler:%w", err)
+ }
+
+ // price
+ priceHandler, err := handler.NewPriceHandler()
+ if err != nil {
+ return nil, fmt.Errorf("error creating price handler:%w", err)
+ }
+
+ // order
+ orderHandler, err := handler.NewOrderHandler()
+ if err != nil {
+ return nil, fmt.Errorf("error creating order handler:%w", err)
+ }
+
+ // recharge
+ rechargeHandler, err := handler.NewRechargeHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating recharge handler:%w", err)
+ }
+
+ apiGroup := r.Group("/api/v1/accounting")
+
+ creditGroup := apiGroup.Group("/credit")
+ {
+ creditGroup.GET("/balance", creditHandler.QueryAllUsersBalance)
+ creditGroup.GET("/:id/balance", creditHandler.QueryBalanceByUserID)
+ creditGroup.GET("/:id/statements", creditHandler.QueryStatementByUserID)
+ creditGroup.GET("/:id/bills", creditHandler.QueryBillsByUserID)
+ creditGroup.PUT("/:id/recharge", creditHandler.RechargeByUserID)
+ creditGroup.PUT("/:id/present", creditHandler.SendPresentByUserID)
+ }
+
+ multiSyncGroup := apiGroup.Group("/multisync")
+ {
+ multiSyncGroup.POST("/quotas", multiSyncHandler.CreateOrUpdateQuota)
+ multiSyncGroup.GET("/quota", multiSyncHandler.QueryQuota)
+ multiSyncGroup.POST("/downloads", multiSyncHandler.CreateQuotaStatement)
+ multiSyncGroup.GET("/download", multiSyncHandler.QueryQuotaStatement)
+ }
+
+ meterGroup := apiGroup.Group("/metering")
+ {
+ meterGroup.GET("/:id/statements", meterHandler.QueryMeteringStatementByUserID)
+ meterGroup.GET("/stat", meterHandler.QueryMeteringStatByDate)
+ }
+
+ priceGroup := apiGroup.Group("/price")
+ {
+ priceGroup.POST("", priceHandler.PriceCreate)
+ priceGroup.GET("/:id", priceHandler.GetPriceByID)
+ priceGroup.PUT("/:id", priceHandler.PriceUpdate)
+ priceGroup.DELETE("/:id", priceHandler.PriceDelete)
+ priceGroup.GET("", priceHandler.QueryPricesBySKUType)
+ }
+
+ orderGroup := apiGroup.Group("/order")
+ {
+ orderGroup.POST("", orderHandler.OrderCreate)
+ orderGroup.GET("/:id", orderHandler.OrderGetByID)
+ }
+
+ rechargeGroup := apiGroup.Group("/recharge")
+ {
+ rechargeGroup.POST("/create-pay-order", rechargeHandler.CreateRecharge)
+ rechargeGroup.GET("/:id/status", rechargeHandler.FetchRechargeStatus)
+ rechargeGroup.GET("/user-recharge-list", rechargeHandler.QueryCurrentUserRechargeList)
+ }
+
+ return r, nil
+}
+
+func createCustomValidator() error {
+ if v, ok := binding.Validator.Engine().(*validator.Validate); ok {
+ return v.RegisterValidation("optional_date_format", OptionalDateFormat)
+ } else {
+ return fmt.Errorf("fail to register custom validation functions")
+ }
+}
+
+
+
package router
+
+import (
+ "time"
+
+ "github.com/go-playground/validator/v10"
+)
+
+func OptionalDateFormat(fl validator.FieldLevel) bool {
+ dateStr := fl.Field().String()
+ if len(dateStr) == 0 {
+ return true
+ }
+ _, err := time.Parse("2006-01-02", dateStr)
+ return err == nil
+}
+
+
+
package utils
+
+import (
+ "time"
+)
+
+func ValidateDateTimeFormat(timeStr, layout string) bool {
+ _, err := time.Parse(layout, timeStr)
+ return err == nil
+}
+
+
+
package utils
+
+import (
+ "fmt"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+)
+
+func GetSceneFromContext(ctx *gin.Context) (int, error) {
+ str := ctx.Query("scene")
+ if str == "" {
+ return 0, fmt.Errorf("bad request scene format")
+ }
+ scene, err := strconv.Atoi(str)
+ return scene, err
+}
+
+
+
package utils
+
+import "opencsg.com/csghub-server/common/types"
+
+func IsNeedCalculateBill(scene types.SceneType) bool {
+ switch scene {
+ case types.SceneModelInference, types.SceneSpace, types.SceneModelFinetune, types.SceneEvaluation, types.SceneStarship:
+ return true
+ default:
+ return false
+ }
+}
+
+func GetSkuUnitTypeByScene(scene types.SceneType) string {
+ switch scene {
+ case types.SceneModelInference:
+ return types.UnitMinute
+ case types.SceneSpace:
+ return types.UnitMinute
+ case types.SceneModelFinetune:
+ return types.UnitMinute
+ case types.SceneMultiSync:
+ return types.UnitRepo
+ case types.SceneEvaluation:
+ return types.UnitMinute
+ case types.SceneStarship:
+ return types.UnitToken
+ default:
+ return types.UnitMinute
+ }
+}
+
+func GetSKUTypeByScene(scene types.SceneType) types.SKUType {
+ switch scene {
+ case types.SceneModelInference:
+ return types.SKUCSGHub
+ case types.SceneSpace:
+ return types.SKUCSGHub
+ case types.SceneModelFinetune:
+ return types.SKUCSGHub
+ case types.SceneMultiSync:
+ return types.SKUCSGHub
+ case types.SceneEvaluation:
+ return types.SKUCSGHub
+ case types.SceneStarship:
+ return types.SKUStarship
+ }
+ return types.SKUReserve
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+type AccountingHandler struct {
+ ac component.AccountingComponent
+ apiToken string
+}
+
+func NewAccountingHandler(config *config.Config) (*AccountingHandler, error) {
+ acctComp, err := component.NewAccountingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &AccountingHandler{
+ ac: acctComp,
+ apiToken: config.APIToken,
+ }, nil
+}
+
+// GetBalances godoc
+// @Security ApiKey
+// @Summary Get all users balance
+// @Description Get all users balance
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current_user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/credit/balance [get]
+func (ah *AccountingHandler) QueryAllUsersBalance(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ data, err := ah.ac.QueryAllUsersBalance(ctx, currentUser, per, page)
+ if err != nil {
+ errTip := "fail to get all accounts balance"
+ slog.Error(errTip, slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// GetUserBalance godoc
+// @Security ApiKey
+// @Summary Get user balance by user uuid
+// @Description Get user balance by user uuid
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path string true "user uuid"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/credit/{id}/balance [get]
+func (ah *AccountingHandler) QueryBalanceByUserID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ userUUID := ctx.Param("id")
+ if len(userUUID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ data, err := ah.ac.QueryBalanceByUserID(ctx, currentUser, userUUID)
+ if err != nil {
+ errTip := "fail to get account balance"
+ slog.Error(errTip, slog.Any("currentUser", currentUser), slog.Any("userUUID", userUUID), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// ListStatement godoc
+// @Security ApiKey
+// @Summary List statements by user uuid and start time and end time
+// @Description List statements by user uuid and start time and end time
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path string true "user uuid"
+// @Param scene query int false "scene" Enums(10, 11, 12, 20) default(10)
+// @Param instance_name query string true "instance name"
+// @Param start_time query string true "start_time, format: '2024-06-12 08:27:22'"
+// @Param end_time query string true "end_time, format: '2024-06-12 17:17:22'"
+// @Param current_user query string true "current_user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/credit/{id}/statements [get]
+func (ah *AccountingHandler) QueryStatementByUserID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ scene, err := getSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userUUID := ctx.Param("id")
+ instance_name := ctx.Query("instance_name")
+ startTime := ctx.Query("start_time") // format: '2024-06-12 08:27:22'
+ endTime := ctx.Query("end_time") // format: '2024-06-12 17:17:22'
+ if len(startTime) < 1 || len(endTime) < 1 || len(userUUID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ if !validateDateTimeFormat(startTime, "2006-01-02 15:04:05") || !validateDateTimeFormat(endTime, "2006-01-02 15:04:05") {
+ slog.Error("Bad request datetime format")
+ httpbase.BadRequest(ctx, "Bad request datetime format")
+ return
+ }
+ req := types.ACCT_STATEMENTS_REQ{
+ CurrentUser: currentUser,
+ UserUUID: userUUID,
+ Scene: scene,
+ InstanceName: instance_name,
+ StartTime: startTime,
+ EndTime: endTime,
+ Per: per,
+ Page: page,
+ }
+ data, err := ah.ac.ListStatementByUserIDAndTime(ctx, req)
+ if err != nil {
+ errTip := "fail to query statement by user"
+ slog.Error(errTip, slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// ListBills godoc
+// @Security ApiKey
+// @Summary List user bills by user uuid and start date and end date
+// @Description List user bills by user uuid and start date and end date
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path string true "user uuid"
+// @Param scene query int false "scene" Enums(10, 11, 12, 20) default(10)
+// @Param start_date query string true "start_date, format: '2024-06-12'"
+// @Param end_date query string true "end_date, format: '2024-07-12'"
+// @Param current_user query string true "current_user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/credit/{id}/bills [get]
+func (ah *AccountingHandler) QueryBillsByUserID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ scene, err := getSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userUUID := ctx.Param("id")
+ startDate := ctx.Query("start_date") // format: '2024-06-12'
+ endDate := ctx.Query("end_date") // format: '2024-06-12'
+ if len(startDate) < 1 || len(endDate) < 1 || len(userUUID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+
+ if !validateDateTimeFormat(startDate, "2006-01-02") || !validateDateTimeFormat(endDate, "2006-01-02") {
+ slog.Error("Bad request date format")
+ httpbase.BadRequest(ctx, "Bad request date format")
+ return
+ }
+ req := types.ACCT_STATEMENTS_REQ{
+ CurrentUser: currentUser,
+ UserUUID: userUUID,
+ Scene: scene,
+ StartTime: startDate,
+ EndTime: endDate,
+ Per: per,
+ Page: page,
+ }
+ data, err := ah.ac.ListBillsByUserIDAndDate(ctx, req)
+ if err != nil {
+ errTip := "fail to query bills by user"
+ slog.Error(errTip, slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// AccountRecharge godoc
+// @Security ApiKey
+// @Summary Recharge fee for account
+// @Description Recharge fee for account
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path string true "user uuid"
+// @Param current_user query string true "current_user"
+// @Param body body types.RECHARGE_REQ true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/credit/{id}/recharge [put]
+func (ah *AccountingHandler) RechargeByUserID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.RECHARGE_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad recharge request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ if req.Value < 0 {
+ slog.Error("Bad recharge value")
+ httpbase.BadRequest(ctx, "Bad recharge value")
+ return
+ }
+ userUUID := ctx.Param("id")
+ if len(userUUID) < 1 {
+ slog.Error("Bad recharge user id")
+ httpbase.BadRequest(ctx, "Bad recharge user id")
+ return
+ }
+
+ data, err := ah.ac.RechargeAccountingUser(ctx, currentUser, userUUID, req)
+ if err != nil {
+ errTip := "fail to recharge user"
+ slog.Error(errTip, slog.Any("currentUser", currentUser), slog.Any("userUUID", userUUID), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+
+ httpbase.OK(ctx, data)
+}
+
+// CreateOrUpdateQuota godoc
+// @Security ApiKey
+// @Summary Add or update account quota
+// @Description Add or update account quota
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current_user"
+// @Param body body types.ACCT_QUOTA_REQ true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/multisync/quotas [post]
+func (ah *AccountingHandler) CreateOrUpdateQuota(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.ACCT_QUOTA_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad quota request format", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ data, err := ah.ac.CreateOrUpdateQuota(currentUser, req)
+ if err != nil {
+ errTip := "fail to add or update account quota"
+ slog.Error(errTip, slog.Any("currentUser", currentUser), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// GetQuota godoc
+// @Security ApiKey
+// @Summary Get account quota by user id
+// @Description Get account quota by user id
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/multisync/quota [get]
+func (ah *AccountingHandler) QueryQuota(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ data, err := ah.ac.GetQuotaByID(currentUser)
+ if err != nil {
+ errTip := "fail to get quota by user"
+ slog.Error(errTip, slog.Any("currentUser", currentUser), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// AddDownload godoc
+// @Security ApiKey
+// @Summary Add download count
+// @Description Add download count
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current_user"
+// @Param body body types.ACCT_QUOTA_STATEMENT_REQ true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/multisync/downloads [post]
+func (ah *AccountingHandler) CreateQuotaStatement(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.ACCT_QUOTA_STATEMENT_REQ
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad quota statement request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ data, err := ah.ac.CreateQuotaStatement(currentUser, req)
+ if err != nil {
+ errTip := "fail to create quota statement by user"
+ slog.Error(errTip, slog.Any("currentUser", currentUser), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// GetQuotaStatement godoc
+// @Security ApiKey
+// @Summary Get account quota statement
+// @Description Get account quota statement
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param repo_path query string true "repo path"
+// @Param repo_type query string true "repo type"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/multisync/download [get]
+func (ah *AccountingHandler) QueryQuotaStatement(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ repoPath := ctx.Query("repo_path")
+ repoType := ctx.Query("repo_type")
+ req := types.ACCT_QUOTA_STATEMENT_REQ{
+ RepoPath: repoPath,
+ RepoType: repoType,
+ }
+
+ data, err := ah.ac.GetQuotaStatement(currentUser, req)
+ if err != nil {
+ errTip := "fail to get account quota statement"
+ slog.Error(errTip, slog.Any("currentUser", currentUser), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+
+ httpbase.OK(ctx, data)
+}
+
+func validateDateTimeFormat(timeStr, layout string) bool {
+ _, err := time.Parse(layout, timeStr)
+ return err == nil
+}
+
+func getSceneFromContext(ctx *gin.Context) (int, error) {
+ str := ctx.Query("scene")
+ if str == "" {
+ return 0, fmt.Errorf("bad request scene format")
+ }
+ scene, err := strconv.Atoi(str)
+ return scene, err
+}
+
+// GetPrices godoc
+// @Security ApiKey
+// @Summary List sku prices
+// @Description List sku prices
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param sku_type query string true "sku_type" Enums(1, 2) default(1)
+// @Param sku_kind query string true "sku_kind"
+// @Param resource_id query string true "resource_id"
+// @Param current_user query string true "current_user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/price [get]
+func (ah *AccountingHandler) QueryPricesBySKUType(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ skuTypeStr := ctx.Query("sku_type")
+ skuTypeInt, err := strconv.Atoi(skuTypeStr)
+ if err != nil {
+ slog.Error("Bad request sku type format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ skuKind := ctx.Query("sku_kind")
+ resID := ctx.Query("resource_id")
+ req := types.AcctPriceListReq{
+ SkuType: types.SKUType(skuTypeInt),
+ SkuKind: skuKind,
+ ResourceID: resID,
+ Per: per,
+ Page: page,
+ }
+ data, err := ah.ac.QueryPricesBySKUType(currentUser, req)
+ if err != nil {
+ slog.Error("fail to query prices by sku type and resource id", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, data)
+}
+
+// GetPrice godoc
+// @Security ApiKey
+// @Summary Get price by id
+// @Description Get price by id
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path int true "id"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/price/{id} [get]
+func (ah *AccountingHandler) GetPriceByID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ idStr := ctx.Param("id")
+ if len(idStr) < 1 {
+ slog.Error("Bad request id format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ id, err := strconv.Atoi(idStr)
+ if err != nil {
+ slog.Error("Bad request id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ data, err := ah.ac.GetPriceByID(currentUser, int64(id))
+ if err != nil {
+ slog.Error("fail to query price by id", slog.Any("id", id), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// CreatePrice godoc
+// @Security ApiKey
+// @Summary Add sku price
+// @Description Add sku price
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current_user"
+// @Param body body types.AcctPriceCreateReq true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/price [post]
+func (ah *AccountingHandler) PriceCreate(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req types.AcctPriceCreateReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad price request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ data, err := ah.ac.CreatePrice(currentUser, req)
+ if err != nil {
+ slog.Error("fail to add price", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, data)
+}
+
+// UpdatePrice godoc
+// @Security ApiKey
+// @Summary Update sku price
+// @Description Update sku price
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path int true "id"
+// @Param current_user query string true "current_user"
+// @Param body body types.AcctPriceCreateReq true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/price/{id} [put]
+func (ah *AccountingHandler) PriceUpdate(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ idStr := ctx.Param("id")
+ if len(idStr) < 1 {
+ slog.Error("Bad request id format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ id, err := strconv.Atoi(idStr)
+ if err != nil {
+ slog.Error("Bad request id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req types.AcctPriceCreateReq
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad price request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ data, err := ah.ac.UpdatePrice(currentUser, req, int64(id))
+ if err != nil {
+ slog.Error("fail to update price", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, data)
+}
+
+// DeletePrice godoc
+// @Security ApiKey
+// @Summary Delete price by id
+// @Description Delete price by id
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path int true "id"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/price/{id} [delete]
+func (ah *AccountingHandler) PriceDelete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ idStr := ctx.Param("id")
+ if len(idStr) < 1 {
+ slog.Error("Bad request id format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ id, err := strconv.Atoi(idStr)
+ if err != nil {
+ slog.Error("Bad request id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ data, err := ah.ac.DeletePrice(currentUser, int64(id))
+ if err != nil {
+ slog.Error("fail to delete price by id", slog.Any("id", id), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// ListMeterings godoc
+// @Security ApiKey
+// @Summary List meterings by user uuid and start time and end time
+// @Description List meterings by user uuid and start time and end time
+// @Tags Accounting
+// @Accept json
+// @Produce json
+// @Param id path string true "user uuid"
+// @Param scene query int false "scene" Enums(10, 11, 12, 20) default(10)
+// @Param instance_name query string true "instance name"
+// @Param start_time query string true "start_time, format: '2024-06-12 08:27:22'"
+// @Param end_time query string true "end_time, format: '2024-06-12 17:17:22'"
+// @Param current_user query string true "current_user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /accounting/metering/{id}/statements [get]
+func (ah *AccountingHandler) QueryMeteringStatementByUserID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ scene, err := getSceneFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request scene format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ userUUID := ctx.Param("id")
+ instance_name := ctx.Query("instance_name")
+ startTime := ctx.Query("start_time") // format: '2024-06-12 08:27:22'
+ endTime := ctx.Query("end_time") // format: '2024-06-12 17:17:22'
+ if len(startTime) < 1 || len(endTime) < 1 || len(userUUID) < 1 {
+ slog.Error("Bad request format")
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ if !validateDateTimeFormat(startTime, "2006-01-02 15:04:05") || !validateDateTimeFormat(endTime, "2006-01-02 15:04:05") {
+ slog.Error("Bad request datetime format")
+ httpbase.BadRequest(ctx, "Bad request datetime format")
+ return
+ }
+ req := types.ACCT_STATEMENTS_REQ{
+ CurrentUser: currentUser,
+ UserUUID: userUUID,
+ Scene: scene,
+ InstanceName: instance_name,
+ StartTime: startTime,
+ EndTime: endTime,
+ Per: per,
+ Page: page,
+ }
+ data, err := ah.ac.ListMeteringsByUserIDAndTime(ctx, req)
+ if err != nil {
+ errTip := "fail to query meterings by user"
+ slog.Error(errTip, slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New(errTip))
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+
+
package callback
+
+import (
+ "log/slog"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "go.temporal.io/sdk/client"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/api/workflow"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ component "opencsg.com/csghub-server/component/callback"
+)
+
+type GitCallbackHandler struct {
+ cbc *component.GitCallbackComponent
+ config *config.Config
+}
+
+func NewGitCallbackHandler(config *config.Config) (*GitCallbackHandler, error) {
+ cbc, err := component.NewGitCallback(config)
+ if err != nil {
+ return nil, err
+ }
+ cbc.SetRepoVisibility(true)
+
+ return &GitCallbackHandler{cbc: cbc, config: config}, nil
+}
+
+func (h *GitCallbackHandler) Handle(c *gin.Context) {
+ event := c.Request.Header.Get("X-Gitea-Event")
+ switch event {
+ case "push":
+ h.handlePush(c)
+ default:
+ slog.Error("Unknown git callback event", "event", event)
+ httpbase.BadRequest(c, "unknown git callback event:"+event)
+ }
+
+}
+
+func (h *GitCallbackHandler) handlePush(c *gin.Context) {
+ var req types.GiteaCallbackPushReq
+ if err := c.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad gitea callback request format", "error", err)
+ c.JSON(http.StatusBadRequest, gin.H{"message": err.Error()})
+ return
+ }
+ //start workflow to handle push request
+ workflowClient := workflow.GetWorkflowClient()
+ workflowOptions := client.StartWorkflowOptions{
+ TaskQueue: workflow.HandlePushQueueName,
+ }
+
+ we, err := workflowClient.ExecuteWorkflow(c, workflowOptions, workflow.HandlePushWorkflow,
+ &req,
+ h.config,
+ )
+ if err != nil {
+ slog.Error("failed to handle git push callback", slog.Any("error", err))
+ httpbase.ServerError(c, err)
+ return
+ }
+
+ slog.Info("start handle push workflow", slog.String("workflow_id", we.GetID()), slog.String("run_id", we.GetRunID()), slog.Any("req", &req))
+ slog.Info("handle git callback successfully", slog.String("event", "push"))
+ httpbase.OK(c, nil)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewClusterHandler(config *config.Config) (*ClusterHandler, error) {
+ ncc, err := component.NewClusterComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &ClusterHandler{
+ c: ncc,
+ }, nil
+}
+
+type ClusterHandler struct {
+ c component.ClusterComponent
+}
+
+// Getclusters godoc
+// @Security ApiKey
+// @Summary Get cluster list
+// @Description Get cluster list
+// @Tags Cluster
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /cluster [get]
+func (h *ClusterHandler) Index(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ clusters, err := h.c.Index(ctx)
+ if err != nil {
+ slog.Error("Failed to get cluster list", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, clusters)
+}
+
+// GetClusterById godoc
+// @Security ApiKey
+// @Summary Get cluster by id
+// @Description Get cluster by id
+// @Tags Cluster
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /cluster/{id} [get]
+func (h *ClusterHandler) GetClusterById(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id := ctx.Param("id")
+ cluster, err := h.c.GetClusterById(ctx, id)
+ if err != nil {
+ slog.Error("Failed to get cluster", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, cluster)
+}
+
+func (h *ClusterHandler) Update(ctx *gin.Context) {
+ var req types.ClusterRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.ClusterID = ctx.Param("id")
+ result, err := h.c.Update(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update cluster info", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, result)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "slices"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewCodeHandler(config *config.Config) (*CodeHandler, error) {
+ tc, err := component.NewCodeComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &CodeHandler{
+ c: tc,
+ sc: sc,
+ }, nil
+}
+
+type CodeHandler struct {
+ c component.CodeComponent
+ sc component.SensitiveComponent
+}
+
+// CreateCode godoc
+// @Security ApiKey
+// @Summary Create a new code
+// @Description create a new code
+// @Tags Code
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user, the owner"
+// @Param body body types.CreateCodeReq true "body"
+// @Success 200 {object} types.Response{data=types.Code} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /codes [post]
+func (h *CodeHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.CreateCodeReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ code, err := h.c.Create(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create code", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create code succeed", slog.String("code", code.Name))
+ respData := gin.H{
+ "data": code,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetVisiableCodes godoc
+// @Security ApiKey
+// @Summary Get Visiable codes for current user
+// @Description get visiable codes for current user
+// @Tags Code
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param search query string false "search text"
+// @Param task_tag query string false "filter by task tag"
+// @Param framework_tag query string false "filter by framework tag"
+// @Param license_tag query string false "filter by license tag"
+// @Param language_tag query string false "filter by language tag"
+// @Param sort query string false "sort by"
+// @Param source query string false "source" Enums(opencsg, huggingface, local)
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Code,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /codes [get]
+func (h *CodeHandler) Index(ctx *gin.Context) {
+ filter := new(types.RepoFilter)
+ filter.Tags = parseTagReqs(ctx)
+ filter.Username = httpbase.GetCurrentUser(ctx)
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filter = getFilterFromContext(ctx, filter)
+ if !slices.Contains[[]string](Sorts, filter.Sort) {
+ msg := fmt.Sprintf("sort parameter must be one of %v", Sorts)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ if filter.Source != "" && !slices.Contains[[]string](Sources, filter.Source) {
+ msg := fmt.Sprintf("source parameter must be one of %v", Sources)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ codes, total, err := h.c.Index(ctx, filter, per, page)
+ if err != nil {
+ slog.Error("Failed to get codes", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get public codes succeed", slog.Int("count", total))
+ respData := gin.H{
+ "data": codes,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// UpdateCode godoc
+// @Security ApiKey
+// @Summary Update a exists code
+// @Description update a exists code
+// @Tags Code
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the owner"
+// @Param body body types.UpdateCodeReq true "body"
+// @Success 200 {object} types.Response{data=database.Code} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /codes/{namespace}/{name} [put]
+func (h *CodeHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdateCodeReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+
+ code, err := h.c.Update(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update code", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Update code succeed", slog.String("code", code.Name))
+ httpbase.OK(ctx, code)
+}
+
+// DeleteCode godoc
+// @Security ApiKey
+// @Summary Delete a exists code
+// @Description delete a exists code
+// @Tags Code
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the owner"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /codes/{namespace}/{name} [delete]
+func (h *CodeHandler) Delete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.Delete(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete code", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Delete code succeed", slog.String("code", name))
+ httpbase.OK(ctx, nil)
+}
+
+// GetCode godoc
+// @Security ApiKey
+// @Summary Get code detail
+// @Description get code detail
+// @Tags Code
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{data=types.Code} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /codes/{namespace}/{name} [get]
+func (h *CodeHandler) Show(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.c.Show(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get code", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get code succeed", slog.String("code", name))
+ httpbase.OK(ctx, detail)
+}
+
+// CodelRelations godoc
+// @Security ApiKey
+// @Summary Get code related assets
+// @Tags Code
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current_user"
+// @Success 200 {object} types.Response{data=types.Relations} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /codes/{namespace}/{name}/relations [get]
+func (h *CodeHandler) Relations(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.c.Relations(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get code relations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "slices"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewCollectionHandler(cfg *config.Config) (*CollectionHandler, error) {
+ cc, err := component.NewCollectionComponent(cfg)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSensitiveComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &CollectionHandler{
+ cc: cc,
+ sc: sc,
+ }, nil
+}
+
+type CollectionHandler struct {
+ cc component.CollectionComponent
+ sc component.SensitiveComponent
+}
+
+// GetCollections godoc
+// @Summary get all collections
+// @Description get all collections
+// @Tags Collection
+// @Param search query string false "search text"
+// @Param sort query string false "sort by" default("trending")
+// @Param per query int false "per" default(50)
+// @Param page query int false "per page" default(1)
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Collection,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections [get]
+func (c *CollectionHandler) Index(ctx *gin.Context) {
+ filter := new(types.CollectionFilter)
+ filter = getCollectionFilter(ctx, filter)
+ if !slices.Contains(types.CollectionSorts, filter.Sort) {
+ msg := fmt.Sprintf("sort parameter must be one of %v", types.CollectionSorts)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ collections, total, err := c.cc.GetCollections(ctx, filter, per, page)
+ if err != nil {
+ slog.Error("Failed to load collections", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": collections,
+ "total": total,
+ }
+
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// CreateCollection godoc
+// @Security JWT token
+// @Summary create a collection
+// @Description create a collection
+// @Tags Collection
+// @Accept json
+// @Produce json
+// @Param body body types.CreateCollectionReq true "body"
+// @Success 200 {object} types.Response{data=types.Collection} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections [post]
+func (c *CollectionHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req *types.CreateCollectionReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := c.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req.Username = currentUser
+ collection, err := c.cc.CreateCollection(ctx, *req)
+ if err != nil {
+ slog.Error("Failed to create collection", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, collection)
+}
+
+// GetCollection godoc
+// @Summary get a collection detail
+// @Description get a collection detail
+// @Tags Collection
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{data=types.Collection} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections/{id} [get]
+func (c *CollectionHandler) GetCollection(ctx *gin.Context) {
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ collection, err := c.cc.GetCollection(ctx, currentUser, id)
+ if err != nil {
+ slog.Error("Failed to create space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, collection)
+}
+
+// UpdateCollection godoc
+// @Security JWT token
+// @Summary update a collection
+// @Description update a collection
+// @Tags Collection
+// @Accept json
+// @Produce json
+// @Param body body types.CreateCollectionReq true "body"
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{data=types.Collection} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections/{id} [put]
+func (c *CollectionHandler) UpdateCollection(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req *types.CreateCollectionReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := c.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.ID = id
+
+ collection, err := c.cc.UpdateCollection(ctx, *req)
+ if err != nil {
+ slog.Error("Failed to create space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, collection)
+}
+
+// DeleteCollection godoc
+// @Security JWT token
+// @Summary Delete a exists collection
+// @Description delete a exists collection
+// @Tags Collection
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections/{id} [delete]
+func (c *CollectionHandler) DeleteCollection(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = c.cc.DeleteCollection(ctx, id, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete collection", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// AddRepoToCollection godoc
+// @Security JWT token
+// @Summary Add repos to a collection
+// @Description Add repos to a collection
+// @Tags Collection
+// @Accept json
+// @Produce json
+// @Param body body types.UpdateCollectionReposReq true "body"
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{data=database.Collection} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections/{id}/repos [post]
+func (c *CollectionHandler) AddRepoToCollection(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req *types.UpdateCollectionReposReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Username = currentUser
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.ID = id
+
+ err = c.cc.AddReposToCollection(ctx, *req)
+ if err != nil {
+ slog.Error("Failed to create collection", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// RemoveRepoFromCollection godoc
+// @Security JWT token
+// @Summary remove repos from a collection
+// @Description remove repos from a collection
+// @Tags Collection
+// @Accept json
+// @Produce json
+// @Param body body types.UpdateCollectionReposReq true "body"
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{data=types.Collection} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /collections/{id}/repos [delete]
+func (c *CollectionHandler) RemoveRepoFromCollection(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req *types.UpdateCollectionReposReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Username = currentUser
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.ID = id
+
+ err = c.cc.RemoveReposFromCollection(ctx, *req)
+ if err != nil {
+ slog.Error("Failed to create collection", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+func getCollectionFilter(ctx *gin.Context, filter *types.CollectionFilter) *types.CollectionFilter {
+ filter.Search = ctx.Query("search")
+ filter.Sort = ctx.Query("sort")
+ if filter.Sort == "" {
+ filter.Sort = "trending"
+ }
+ return filter
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/proxy"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+type DataflowProxyHandler struct {
+ rp *proxy.ReverseProxy
+ user component.UserComponent
+ usc rpc.UserSvcClient
+}
+
+func NewDataflowProxyHandler(config *config.Config) (*DataflowProxyHandler, error) {
+ uc, err := component.NewUserComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create user component: %w", err)
+ }
+ remoteEndpoint := fmt.Sprintf("%s:%d", config.Dataflow.Host, config.Dataflow.Port)
+ rp, err := proxy.NewReverseProxy(remoteEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create dataflow reverse proxy: %w", err)
+ }
+ usc := rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+ return &DataflowProxyHandler{rp: rp, user: uc, usc: usc}, nil
+}
+
+// Proxy send request to backend service, without change the request path
+func (h *DataflowProxyHandler) Proxy(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ u, err := h.user.GetUserByName(ctx, currentUser)
+ if err != nil {
+ httpbase.UnauthorizedError(ctx, err)
+ ctx.Abort()
+ return
+ }
+ token, err := h.usc.GetOrCreateFirstAvaiTokens(ctx, currentUser, currentUser, string(types.AccessTokenAppGit), "dataflow")
+ if err != nil {
+
+ httpbase.ServerError(ctx, err)
+ ctx.Abort()
+ return
+ }
+ if len(token) == 0 {
+ slog.Error("fail to get or create user first git access token", slog.Any("user", currentUser), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New("can not get user first available access token"))
+ ctx.Abort()
+ return
+ }
+ ctx.Request.Header.Set("user_id", fmt.Sprintf("%d", u.ID))
+ ctx.Request.Header.Set("user_name", u.Username)
+ ctx.Request.Header.Set("user_token", token)
+ ctx.Request.Header.Set("isadmin", fmt.Sprintf("%t", u.CanAdmin()))
+ // Log the request URL and header
+ slog.Debug("http request", slog.Any("request", ctx.Request.URL), slog.Any("header", ctx.Request.Header))
+ // Serve the request using the router
+ h.rp.ServeHTTP(ctx.Writer, ctx.Request, "")
+}
+
+// ProxyToApi similar with Proxy, but change the request path
+//
+// the target request path can read params from the origin request path. This request will read 'id' from original request:
+//
+// apiGroup.PUT("/dataflow/job/get/:id", userProxyHandler.ProxyToApi("/api/v1/dataflow/job/get/%v", "id"))
+func (h *DataflowProxyHandler) ProxyToApi(api string, originParams ...string) gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ slog.Info("proxy dataflow request", slog.Any("request", ctx.Request.URL), slog.Any("header", ctx.Request.Header))
+ finalApi := api
+ if len(originParams) > 0 {
+ var params []any
+ for _, op := range originParams {
+ params = append(params, ctx.Param(op))
+ }
+ finalApi = fmt.Sprintf(finalApi, params...)
+ }
+ h.rp.ServeHTTP(ctx.Writer, ctx.Request, finalApi)
+ }
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "slices"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+var Sorts = []string{"trending", "recently_update", "most_download", "most_favorite"}
+var Sources = []string{"opencsg", "huggingface", "local"}
+
+func NewDatasetHandler(config *config.Config) (*DatasetHandler, error) {
+ tc, err := component.NewDatasetComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ repo, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating repo component:%w", err)
+ }
+ return &DatasetHandler{
+ c: tc,
+ sc: sc,
+ repo: repo,
+ }, nil
+}
+
+type DatasetHandler struct {
+ c component.DatasetComponent
+ sc component.SensitiveComponent
+ repo component.RepoComponent
+}
+
+// CreateDataset godoc
+// @Security ApiKey
+// @Summary Create a new dataset
+// @Description create a new dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user, the owner"
+// @Param body body types.CreateDatasetReq true "body"
+// @Success 200 {object} types.Response{data=types.Dataset} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets [post]
+func (h *DatasetHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.CreateDatasetReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ // don't allow to create public dataset
+ // datasets must pass the sensitive check then it can be public
+ if !req.Private {
+ httpbase.BadRequest(ctx, "creating public dataset is not allowed")
+ return
+ }
+ req.Username = currentUser
+
+ dataset, err := h.c.Create(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create dataset succeed", slog.String("dataset", dataset.Name))
+ respData := gin.H{
+ "data": dataset,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetVisiableDatasets godoc
+// @Security ApiKey
+// @Summary Get Visiable datasets for current user
+// @Description get visiable datasets for current user
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param search query string false "search text"
+// @Param task_tag query string false "filter by task tag"
+// @Param framework_tag query string false "filter by framework tag"
+// @Param license_tag query string false "filter by license tag"
+// @Param language_tag query string false "filter by language tag"
+// @Param sort query string false "sort by"
+// @Param source query string false "source" Enums(opencsg, huggingface, local)
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Dataset,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets [get]
+func (h *DatasetHandler) Index(ctx *gin.Context) {
+ filter := new(types.RepoFilter)
+ filter.Tags = parseTagReqs(ctx)
+ filter.Username = httpbase.GetCurrentUser(ctx)
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filter = getFilterFromContext(ctx, filter)
+ if !slices.Contains[[]string](Sorts, filter.Sort) {
+ msg := fmt.Sprintf("sort parameter must be one of %v", Sorts)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ if filter.Source != "" && !slices.Contains[[]string](Sources, filter.Source) {
+ msg := fmt.Sprintf("source parameter must be one of %v", Sources)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ datasets, total, err := h.c.Index(ctx, filter, per, page)
+ if err != nil {
+ slog.Error("Failed to get datasets", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get public datasets succeed", slog.Int("count", total))
+ respData := gin.H{
+ "data": datasets,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// UpdateDataset godoc
+// @Security ApiKey
+// @Summary Update a exists dataset
+// @Description update a exists dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the owner"
+// @Param body body types.UpdateDatasetReq true "body"
+// @Success 200 {object} types.Response{data=database.Dataset} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name} [put]
+func (h *DatasetHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdateDatasetReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+
+ dataset, err := h.c.Update(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Update dataset succeed", slog.String("dataset", dataset.Name))
+ httpbase.OK(ctx, dataset)
+}
+
+// DeleteDataset godoc
+// @Security ApiKey
+// @Summary Delete a exists dataset
+// @Description delete a exists dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the owner"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name} [delete]
+func (h *DatasetHandler) Delete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.Delete(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Delete dataset succeed", slog.String("dataset", name))
+ httpbase.OK(ctx, nil)
+}
+
+// GetDataset godoc
+// @Security ApiKey
+// @Summary Get dataset detail
+// @Description get dataset detail
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{data=types.Dataset} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name} [get]
+func (h *DatasetHandler) Show(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.c.Show(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get dataset succeed", slog.String("dataset", name))
+ httpbase.OK(ctx, detail)
+}
+
+// DatasetRelations godoc
+// @Security ApiKey
+// @Summary Get dataset related assets
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current_user"
+// @Success 200 {object} types.Response{data=types.Relations} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name}/relations [get]
+func (h *DatasetHandler) Relations(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.c.Relations(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get dataset relations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+func getFilterFromContext(ctx *gin.Context, filter *types.RepoFilter) *types.RepoFilter {
+ filter.Search = ctx.Query("search")
+ filter.Sort = ctx.Query("sort")
+ if filter.Sort == "" {
+ filter.Sort = "recently_update"
+ }
+ filter.Source = ctx.Query("source")
+ return filter
+}
+
+// DatasetFiles godoc
+// @Security ApiKey
+// @Summary Get all files of a dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{data=types.File} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name}/all_files [get]
+func (h *DatasetHandler) AllFiles(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req types.GetAllFilesReq
+ req.Namespace = namespace
+ req.Name = name
+ req.RepoType = types.DatasetRepo
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Ref = ""
+ detail, err := h.repo.AllFiles(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get dataset all files", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "regexp"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+type DatasetViewerHandler struct {
+ c component.DatasetViewerComponent
+}
+
+func NewDatasetViewerHandler(cfg *config.Config) (*DatasetViewerHandler, error) {
+ dvc, err := component.NewDatasetViewerComponent(cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ return &DatasetViewerHandler{
+ c: dvc,
+ }, nil
+}
+
+// GetDatasetDemoData godoc
+// @Security ApiKey
+// @Summary Get the demo data of the dataset
+// @Description get the demo data of the dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Parsm name path string true "name"
+// @Param file_path path string true "file_path"
+// @Param per query int false "per" default(50)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name}/viewer/{file_path} [get]
+func (h *DatasetViewerHandler) View(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad repo request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad paging request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req := new(component.ViewParquetFileReq)
+ req.Namespace = namespace
+ req.RepoName = name
+ req.Path = ctx.Param("file_path")
+ req.Per = per
+ req.Page = page
+ req.CurrentUser = currentUser
+
+ resp, err := h.c.ViewParquetFile(ctx, req)
+ if err != nil {
+ slog.Error("Failed to view parquet file", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, resp)
+}
+
+// GetDatasetCatalog godoc
+// @Security ApiKey
+// @Summary Get catalog of the dataset
+// @Description get catalog of the dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Parsm name path string true "name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name}/dataviewer/catalog [get]
+func (h *DatasetViewerHandler) Catalog(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req := new(component.ViewParquetFileReq)
+ req.Namespace = namespace
+ req.RepoName = name
+ req.Branch = "main"
+ req.CurrentUser = currentUser
+
+ catalog, err := h.c.GetCatalog(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get dataset catalog", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, catalog)
+}
+
+// GetDatasetCatalog godoc
+// @Security ApiKey
+// @Summary Get catalog of the dataset
+// @Description get catalog of the dataset
+// @Tags Dataset
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Parsm name path string true "name"
+// @Param config query string true "config"
+// @Param split query string true "split"
+// @Param search query string false "search"
+// @Param where query string false "where"
+// @Param orderby query string false "orderby"
+// @Param per query int false "per" default(50)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /datasets/{namespace}/{name}/dataviewer/rows [get]
+func (h *DatasetViewerHandler) Rows(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad repo request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad paging request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ // config=SelfRC&split=train
+ config := ctx.Query("config")
+ split := ctx.Query("split")
+ search := ctx.Query("search")
+ where := ctx.Query("where")
+ orderby := ctx.Query("orderby")
+ if config == "" || split == "" {
+ slog.Error("Bad view rows request format")
+ httpbase.BadRequest(ctx, "Bad view rows request format")
+ return
+ }
+ var viewReq types.DataViewerReq
+ viewReq.Config = config
+ viewReq.Split = split
+ viewReq.Search = search
+ viewReq.Where = where
+ viewReq.Orderby = orderby
+
+ req := new(component.ViewParquetFileReq)
+ req.Namespace = namespace
+ req.RepoName = name
+ req.Branch = "main"
+ req.CurrentUser = currentUser
+ req.Per = per
+ req.Page = page
+
+ slog.Debug("hander.rows viewerReq", slog.Any("viewReq", viewReq))
+ err = validateQueryParameter(where, "where")
+ if err != nil {
+ slog.Error("invalid character in parameter where", slog.Any("req", req), slog.Any("viewReq", viewReq), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ err = validateQueryParameter(orderby, "orderby")
+ if err != nil {
+ slog.Error("invalid character in parameter orderby", slog.Any("req", req), slog.Any("viewReq", viewReq), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ err = validateQueryParameter(search, "search")
+ if err != nil {
+ slog.Error("invalid character in parameter search", slog.Any("req", req), slog.Any("viewReq", viewReq), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ rows, err := h.c.Rows(ctx, req, viewReq)
+ if err != nil {
+ slog.Error("Failed to get dataset rows", slog.Any("req", req), slog.Any("viewReq", viewReq), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, rows)
+}
+
+func validateQueryParameter(parameterValue string, parameterName string) error {
+ SQLInvalidSymbols := []string{";", "--", `/\*`, `\*/`}
+ SQLInvalidSymbolsPattern := regexp.MustCompile(fmt.Sprintf("(?:%s)", strings.Join(SQLInvalidSymbols, "|")))
+
+ if SQLInvalidSymbolsPattern.MatchString(parameterValue) {
+ return fmt.Errorf("invalid character in %s", parameterName)
+ }
+ return nil
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+type DiscussionHandler struct {
+ c component.DiscussionComponent
+ sc component.SensitiveComponent
+}
+
+func NewDiscussionHandler(cfg *config.Config) (*DiscussionHandler, error) {
+ c := component.NewDiscussionComponent()
+ sc, err := component.NewSensitiveComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create sensitive component: %w", err)
+ }
+ return &DiscussionHandler{
+ c: c,
+ sc: sc,
+ }, nil
+}
+
+// CreateRepoDiscussion godoc
+// @Security ApiKey
+// @Summary Create a new repo discussion
+// @Description create a new repo discussion
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current user, the owner"
+// @Param repo_type path string true "repository type" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param body body component.CreateRepoDiscussionRequest true "body"
+// @Success 200 {object} types.Response{data=component.CreateDiscussionResponse} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/discussions [post]
+func (h *DiscussionHandler) CreateRepoDiscussion(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ repoType := h.getRepoType(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req component.CreateRepoDiscussionRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req.CurrentUser = currentUser
+ req.RepoType = types.RepositoryType(repoType)
+ req.Namespace = namespace
+ req.Name = name
+ resp, err := h.c.CreateRepoDiscussion(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create repo discussion", "error", err, "request", req)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to create repo discussion: %w", err))
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// UpdateDiscussion godoc
+// @Security ApiKey
+// @Summary Update a discussion
+// @Description update a discussion
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the discussion id"
+// @Param current_user query string true "current user, the owner"
+// @Param body body component.UpdateDiscussionRequest true "body"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /discussions/{id} [put]
+func (h *DiscussionHandler) UpdateDiscussion(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ httpbase.BadRequest(ctx, "invalid discussion id:"+id)
+ return
+ }
+ var req component.UpdateDiscussionRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req.ID = idInt
+ req.CurrentUser = currentUser
+ err = h.c.UpdateDiscussion(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update discussion", "error", err, "request", req)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to update discussion: %w", err))
+ return
+ }
+ httpbase.OK(ctx, nil)
+
+}
+
+// DeleteDiscussion godoc
+// @Security ApiKey
+// @Summary Delete a discussion
+// @Description delete a discussion
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the discussion id"
+// @Param current_user query string true "current user, the owner of the discussion"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /discussions/{id} [delete]
+func (h *DiscussionHandler) DeleteDiscussion(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.DeleteDiscussion(ctx, currentUser, idInt)
+ if err != nil {
+ slog.Error("Failed to delete discussion", "error", err, "id", id)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to delete discussion: %w", err))
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// ShowDiscussion godoc
+// @Security ApiKey
+// @Summary Show a discussion and its comments
+// @Description show a discussion
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the discussion id"
+// @Success 200 {object} types.Response{data=component.ShowDiscussionResponse} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /discussions/{id} [get]
+func (h *DiscussionHandler) ShowDiscussion(ctx *gin.Context) {
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ d, err := h.c.GetDiscussion(ctx, idInt)
+ if err != nil {
+ slog.Error("Failed to get discussion", "error", err, "id", id)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to get discussion: %w", err))
+ return
+ }
+ httpbase.OK(ctx, d)
+}
+
+// ListRepoDiscussions godoc
+// @Security ApiKey
+// @Summary List repo discussions
+// @Description list repo discussions
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param repo_type path string true "repository type" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "namespace"
+// @Param name query string true "name"
+// @Success 200 {object} types.Response{data=component.ListRepoDiscussionResponse} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/discussions [get]
+func (h *DiscussionHandler) ListRepoDiscussions(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ repoType := h.getRepoType(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("failed to get namespace and name from request context: %w", err).Error())
+ return
+ }
+
+ var req component.ListRepoDiscussionRequest
+ req.CurrentUser = currentUser
+ req.RepoType = types.RepositoryType(repoType)
+ req.Namespace = namespace
+ req.Name = name
+ resp, err := h.c.ListRepoDiscussions(ctx, req)
+ if err != nil {
+ slog.Error("Failed to list repo discussions", "error", err, "request", req)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to list repo discussions: %w", err))
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// CreateDiscussionComment godoc
+// @Security ApiKey
+// @Summary Create a new discussion comment
+// @Description create a new discussion comment
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the discussion id"
+// @Param body body component.CreateCommentRequest true "body"
+// @Success 200 {object} types.Response{data=component.CreateCommentResponse} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /discussions/{id}/comments [post]
+func (h *DiscussionHandler) CreateDiscussionComment(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("invalid discussion id: %w", err).Error())
+ return
+ }
+ var req component.CreateCommentRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req.CommentableID = idInt
+ req.CurrentUser = currentUser
+
+ resp, err := h.c.CreateDiscussionComment(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create discussion comment", "error", err, "request", req)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to create discussion comment: %w", err))
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// UpdateComment godoc
+// @Security ApiKey
+// @Summary Update a comment content by id
+// @Description update a comment content by id
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the comment id"
+// @Param current_user query string true "current user, the owner of the comment"
+// @Param body body component.UpdateCommentRequest true "body"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /comments/{id} [put]
+func (h *DiscussionHandler) UpdateComment(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("invalid comment id: %w", err).Error())
+ return
+ }
+
+ var req component.UpdateCommentRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ err = h.c.UpdateComment(ctx, currentUser, idInt, req.Content)
+ if err != nil {
+ slog.Error("Failed to update comment", "error", err, "request", req)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to update comment: %w", err))
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteDiscussionComment godoc
+// @Security ApiKey
+// @Summary Delete a comment by id
+// @Description delete a comment by id
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the comment id"
+// @Param current_user query string true "current user, the owner of the comment"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /comments/{id} [delete]
+func (h *DiscussionHandler) DeleteComment(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("invalid comment id: %w", err).Error())
+ return
+ }
+ err = h.c.DeleteComment(ctx, currentUser, idInt)
+ if err != nil {
+ slog.Error("Failed to delete comment", "error", err, "id", id)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to delete comment: %w", err))
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// ListDiscussionComments godoc
+// @Security ApiKey
+// @Summary List discussion comments
+// @Description list discussion comments
+// @Tags Discussion
+// @Accept json
+// @Produce json
+// @Param id path string true "the discussion id"
+// @Success 200 {object} types.Response{data=[]component.DiscussionResponse_Comment} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /discussions/{id}/comments [get]
+func (h *DiscussionHandler) ListDiscussionComments(ctx *gin.Context) {
+ id := ctx.Param("id")
+ idInt, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("invalid discussion id: %w", err).Error())
+ }
+ comments, err := h.c.ListDiscussionComments(ctx, idInt)
+ if err != nil {
+ slog.Error("Failed to list discussion comments", "error", err, "id", id)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to list discussion comments: %w", err))
+ return
+ }
+ httpbase.OK(ctx, comments)
+}
+
+func (h *DiscussionHandler) getRepoType(ctx *gin.Context) types.RepositoryType {
+ repoType := ctx.Param("repo_type")
+ repoType = strings.TrimRight(repoType, "s")
+ return types.RepositoryType(repoType)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewEvaluationHandler(config *config.Config) (*EvaluationHandler, error) {
+ wkf, err := component.NewEvaluationComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &EvaluationHandler{
+ c: wkf,
+ sc: sc,
+ }, nil
+}
+
+type EvaluationHandler struct {
+ c component.EvaluationComponent
+ sc component.SensitiveComponent
+}
+
+// create evaluation godoc
+// @Security ApiKey
+// @Summary run model evaluation
+// @Tags Evaluation
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param body body types.EvaluationReq true "body setting of evaluation"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /evaluations [post]
+func (h *EvaluationHandler) RunEvaluation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ var req types.EvaluationReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err := h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+ evaluation, err := h.c.CreateEvaluation(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create evaluation job", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, evaluation)
+}
+
+// get evaluation godoc
+// @Security ApiKey
+// @Summary get model evaluation
+// @Tags Evaluation
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Success 200 {object} types.EvaluationRes "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /evaluations/{id} [get]
+func (h *EvaluationHandler) GetEvaluation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req = &types.EvaluationGetReq{}
+ req.ID = id
+ req.Username = currentUser
+ evaluation, err := h.c.GetEvaluation(ctx, *req)
+ if err != nil {
+ slog.Error("Failed to get evaluation job", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, evaluation)
+
+}
+
+// deleteEvaluation godoc
+// @Security ApiKey
+// @Summary delete model evaluation
+// @Tags Evaluation
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /evaluations/{id} [delete]
+func (h *EvaluationHandler) DeleteEvaluation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req = &types.EvaluationDelReq{}
+ req.ID = id
+ req.Username = currentUser
+ err = h.c.DeleteEvaluation(ctx, *req)
+ if err != nil {
+ slog.Error("Failed to delete evaluation job", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+type EventHandler struct {
+ ec component.EventComponent
+}
+
+func NewEventHandler() (*EventHandler, error) {
+ return &EventHandler{
+ ec: component.NewEventComponent(),
+ }, nil
+}
+
+// @Summary Report client events
+// @Tags Events
+// @Accept json
+// @Produce json
+// @Param events body []types.Event true "Events"
+// @Success 200 {object} object "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /events [post]
+func (h *EventHandler) Create(ctx *gin.Context) {
+ //TODO: authentication?
+
+ var events []types.Event
+ if err := ctx.ShouldBindJSON(&events); err != nil {
+ err = fmt.Errorf("cant parse as Event array,%w", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if err := h.ec.NewEvents(ctx, events); err != nil {
+ slog.Error("Failed to create events", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "compress/gzip"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "path/filepath"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "github.com/golang/gddo/httputil"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewGitHTTPHandler(config *config.Config) (*GitHTTPHandler, error) {
+ uc, err := component.NewGitHTTPComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &GitHTTPHandler{
+ c: uc,
+ }, nil
+}
+
+type GitHTTPHandler struct {
+ c component.GitHTTPComponent
+}
+
+func (h *GitHTTPHandler) InfoRefs(ctx *gin.Context) {
+ rpc := ctx.Query("service")
+ if !(rpc == "git-upload-pack" || rpc == "git-receive-pack") {
+ httpbase.NotFoundError(ctx, errors.New("service not found"))
+ }
+ ctx.Header("Content-Type", fmt.Sprintf("application/x-%s-advertisement", rpc))
+ ctx.Header("Cache-Control", "no-cache")
+
+ gitProtocol := ctx.GetHeader("Git-Protocol")
+
+ offers := []string{"gzip", "identity"}
+ encoding := httputil.NegotiateContentEncoding(ctx.Request, offers)
+ req := types.InfoRefsReq{
+ Namespace: ctx.GetString("namespace"),
+ Name: ctx.GetString("name"),
+ RepoType: types.RepositoryType(ctx.GetString("repo_type")),
+ Rpc: rpc,
+ GitProtocol: gitProtocol,
+ CurrentUser: httpbase.GetCurrentUser(ctx),
+ }
+ reader, err := h.c.InfoRefs(ctx, req)
+ if err != nil {
+ if err == component.ErrUnauthorized {
+ ctx.Header("WWW-Authenticate", "Basic realm=opencsg-git")
+ ctx.PureJSON(http.StatusUnauthorized, nil)
+ return
+ }
+
+ if err == component.ErrForbidden {
+ ctx.PureJSON(http.StatusForbidden, gin.H{
+ "error": "You do not have permission to access this repository.",
+ })
+ return
+ }
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ if encoding == "gzip" {
+ gzWriter := gzip.NewWriter(ctx.Writer)
+ defer gzWriter.Close()
+
+ gzRespWriter := &gzipResponseWriter{
+ ResponseWriter: ctx.Writer,
+ writer: gzWriter,
+ }
+ ctx.Writer = gzRespWriter
+ ctx.Header("Content-Encoding", "gzip")
+ }
+
+ if _, err = io.Copy(ctx.Writer, reader); err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+}
+
+// func (h *GitHTTPHandler) HEAD(ctx *gin.Context) {
+// httpbase.OK(ctx, nil)
+// }
+
+func (h *GitHTTPHandler) GitUploadPack(ctx *gin.Context) {
+ gitProtocol := ctx.GetHeader("Git-Protocol")
+ req := types.GitUploadPackReq{
+ Namespace: ctx.GetString("namespace"),
+ Name: ctx.GetString("name"),
+ RepoType: types.RepositoryType(ctx.GetString("repo_type")),
+ GitProtocol: gitProtocol,
+ Request: ctx.Request,
+ Writer: ctx.Writer,
+ CurrentUser: httpbase.GetCurrentUser(ctx),
+ }
+ action := getService(ctx.Request)
+
+ ctx.Header("Content-Type", fmt.Sprintf("application/x-%s-result", action))
+ ctx.Header("Cache-Control", "no-cache")
+
+ err := h.c.GitUploadPack(ctx, req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+}
+
+func (h *GitHTTPHandler) GitReceivePack(ctx *gin.Context) {
+ gitProtocol := ctx.GetHeader("Git-Protocol")
+ req := types.GitUploadPackReq{
+ Namespace: ctx.GetString("namespace"),
+ Name: ctx.GetString("name"),
+ RepoType: types.RepositoryType(ctx.GetString("repo_type")),
+ GitProtocol: gitProtocol,
+ Request: ctx.Request,
+ Writer: ctx.Writer,
+ CurrentUser: httpbase.GetCurrentUser(ctx),
+ }
+ action := getService(ctx.Request)
+
+ ctx.Header("Content-Type", fmt.Sprintf("application/x-%s-result", action))
+ ctx.Header("Cache-Control", "no-cache")
+
+ err := h.c.GitReceivePack(ctx, req)
+ if err != nil {
+ if err == component.ErrUnauthorized {
+ ctx.Header("WWW-Authenticate", "Basic realm=opencsg-git")
+ ctx.PureJSON(http.StatusUnauthorized, nil)
+ return
+ }
+
+ if err == component.ErrForbidden {
+ ctx.PureJSON(http.StatusForbidden, gin.H{
+ "error": "You do not have permission to access this repository.",
+ })
+ return
+ }
+ httpbase.ServerError(ctx, err)
+ return
+ }
+}
+
+func (h *GitHTTPHandler) LfsBatch(ctx *gin.Context) {
+ var batchRequest types.BatchRequest
+ if err := ctx.ShouldBindJSON(&batchRequest); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ batchRequest.CurrentUser = httpbase.GetCurrentUser(ctx)
+ batchRequest.Authorization = ctx.Request.Header.Get("Authorization")
+ batchRequest.Namespace = ctx.GetString("namespace")
+ batchRequest.Name = ctx.GetString("name")
+ batchRequest.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+
+ var isUpload bool
+ if batchRequest.Operation == "upload" {
+ isUpload = true
+ } else if batchRequest.Operation == "download" {
+ isUpload = false
+ } else {
+ slog.Error("Invalid lfs batch operation", slog.String("operation", batchRequest.Operation))
+ httpbase.BadRequest(ctx, fmt.Sprintf("Invalid lfs batch operation: %s", batchRequest.Operation))
+ return
+ }
+
+ objectResponse, err := h.c.BuildObjectResponse(ctx, batchRequest, isUpload)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ ctx.Header("WWW-Authenticate", "Basic realm=opencsg-git")
+ ctx.PureJSON(http.StatusUnauthorized, nil)
+ return
+ }
+
+ if errors.Is(err, component.ErrForbidden) {
+ ctx.PureJSON(http.StatusForbidden, gin.H{
+ "error": "You do not have permission to access this repository.",
+ })
+ return
+ }
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.Header("Content-Type", types.LfsMediaType)
+ ctx.PureJSON(http.StatusOK, objectResponse)
+}
+
+func (h *GitHTTPHandler) LfsUpload(ctx *gin.Context) {
+ var err error
+ var uploadRequest types.UploadRequest
+ uploadRequest.Oid = ctx.Param("oid")
+ uploadRequest.Size, err = strconv.ParseInt(ctx.Param("size"), 10, 64)
+ if err != nil {
+ slog.Error("Invalid lfs file size", slog.String("size", ctx.Param("size")))
+ httpbase.BadRequest(ctx, fmt.Sprintf("Invalid lfs file size: %s", ctx.Param("size")))
+ return
+ }
+ uploadRequest.Namespace = ctx.GetString("namespace")
+ uploadRequest.Name = ctx.GetString("name")
+ uploadRequest.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ uploadRequest.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ err = h.c.LfsUpload(ctx, ctx.Request.Body, uploadRequest)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.Header("Content-Type", types.LfsMediaType)
+}
+
+func (h *GitHTTPHandler) LfsDownload(ctx *gin.Context) {
+ var err error
+ var downloadRequest types.DownloadRequest
+ downloadRequest.Oid = ctx.Param("oid")
+ downloadRequest.Size, err = strconv.ParseInt(ctx.Param("size"), 10, 64)
+ if err != nil {
+ slog.Error("Invalid lfs file size", slog.String("size", ctx.Param("size")))
+ httpbase.BadRequest(ctx, fmt.Sprintf("Invalid lfs file size: %s", ctx.Param("size")))
+ return
+ }
+ downloadRequest.Namespace = ctx.GetString("namespace")
+ downloadRequest.Name = ctx.GetString("name")
+ downloadRequest.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ downloadRequest.CurrentUser = httpbase.GetCurrentUser(ctx)
+ downloadRequest.SaveAs = ctx.Query("save_as")
+
+ url, err := h.c.LfsDownload(ctx, downloadRequest)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.Redirect(http.StatusFound, url.String())
+}
+
+func (h *GitHTTPHandler) LfsVerify(ctx *gin.Context) {
+ var (
+ pointer types.Pointer
+ verifyRequest types.VerifyRequest
+ )
+ if err := ctx.ShouldBindJSON(&pointer); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ verifyRequest.Namespace = ctx.GetString("namespace")
+ verifyRequest.Name = ctx.GetString("name")
+ verifyRequest.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ verifyRequest.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ err := h.c.LfsVerify(ctx, verifyRequest, pointer)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ctx.PureJSON(http.StatusOK, nil)
+}
+
+func (h *GitHTTPHandler) ListLocks(ctx *gin.Context) {
+ var (
+ req types.ListLFSLockReq
+ err error
+ )
+
+ req.Namespace = ctx.GetString("namespace")
+ req.Name = ctx.GetString("name")
+ req.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Path = ctx.Query("path")
+ id := ctx.Query("id")
+ if id != "" {
+ req.ID, err = strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusBadRequest, types.LFSLockError{
+ Message: "unable to delete lock : Invalid request",
+ })
+ return
+ }
+ }
+ cursor, _ := strconv.Atoi(ctx.Query("cursor"))
+ if cursor < 0 {
+ cursor = 0
+ }
+ req.Cursor = cursor
+
+ limit, _ := strconv.Atoi(ctx.Query("limit"))
+ if limit < 0 {
+ limit = 0
+ }
+ req.Limit = limit
+
+ res, err := h.c.ListLocks(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ ctx.Header("WWW-Authenticate", "Basic realm=opencsg-git")
+ ctx.PureJSON(http.StatusUnauthorized, types.LFSLockError{
+ Message: "You must have access to read locks",
+ })
+ return
+ }
+ ctx.JSON(http.StatusInternalServerError, types.LFSLockError{
+ Message: "unable to list locks : Internal Server Error",
+ })
+ return
+ }
+ ctx.PureJSON(http.StatusOK, res)
+}
+
+func (h *GitHTTPHandler) CreateLock(ctx *gin.Context) {
+ var req types.LfsLockReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusBadRequest, types.LFSLockError{
+ Message: "unable to delete lock : Invalid request",
+ })
+ return
+ }
+
+ req.Namespace = ctx.GetString("namespace")
+ req.Name = ctx.GetString("name")
+ req.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ lock, err := h.c.CreateLock(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrAlreadyExists) {
+ ctx.PureJSON(http.StatusConflict, types.LFSLockError{
+ Lock: &types.LFSLock{
+ ID: strconv.FormatInt(lock.ID, 10),
+ Path: lock.Path,
+ LockedAt: lock.CreatedAt,
+ Owner: &types.LFSLockOwner{
+ Name: lock.User.Username,
+ },
+ },
+ Message: "already created lock",
+ })
+ return
+ }
+
+ if errors.Is(err, component.ErrUnauthorized) {
+ ctx.Header("WWW-Authenticate", "Basic realm=opencsg-git")
+ ctx.PureJSON(http.StatusUnauthorized, types.LFSLockError{
+ Message: "You must have push access to create locks",
+ })
+ return
+ }
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusInternalServerError, types.LFSLockError{
+ Message: "unable to delete lock : Internal Server Error",
+ })
+ return
+ }
+ ctx.PureJSON(http.StatusOK, &types.LFSLockResponse{
+ Lock: &types.LFSLock{
+ ID: strconv.FormatInt(lock.ID, 10),
+ Path: lock.Path,
+ LockedAt: lock.CreatedAt,
+ Owner: &types.LFSLockOwner{
+ Name: lock.User.Username,
+ },
+ },
+ })
+}
+
+func (h *GitHTTPHandler) VerifyLock(ctx *gin.Context) {
+ var req types.VerifyLFSLockReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusBadRequest, types.LFSLockError{
+ Message: "unable to delete lock : Invalid request",
+ })
+ return
+ }
+
+ req.Namespace = ctx.GetString("namespace")
+ req.Name = ctx.GetString("name")
+ req.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ cursor, _ := strconv.Atoi(ctx.PostForm("cursor"))
+ if cursor < 0 {
+ cursor = 0
+ }
+ req.Cursor = cursor
+
+ limit, _ := strconv.Atoi(ctx.PostForm("limit"))
+ if limit < 0 {
+ limit = 0
+ }
+ req.Limit = limit
+
+ res, err := h.c.VerifyLock(ctx, req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusInternalServerError, types.LFSLockError{
+ Message: "unable to delete lock : Internal Server Error",
+ })
+ return
+ }
+ ctx.PureJSON(http.StatusOK, res)
+}
+
+func (h *GitHTTPHandler) UnLock(ctx *gin.Context) {
+ var (
+ req types.UnlockLFSReq
+ err error
+ lock *database.LfsLock
+ )
+
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusInternalServerError, types.LFSLockError{
+ Message: "unable to delete lock : Bad request format",
+ })
+ return
+ }
+ req.ID, err = strconv.ParseInt(ctx.Param("lid"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusBadRequest, types.LFSLockError{
+ Message: "unable to delete lock : Invalid id",
+ })
+ return
+ }
+
+ req.Namespace = ctx.GetString("namespace")
+ req.Name = ctx.GetString("name")
+ req.RepoType = types.RepositoryType(ctx.GetString("repo_type"))
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ lock, err = h.c.UnLock(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ ctx.Header("WWW-Authenticate", "Basic realm=opencsg-git")
+ ctx.PureJSON(http.StatusUnauthorized, types.LFSLockError{
+ Message: "You must have push access to create locks",
+ })
+ return
+ }
+ if errors.Is(err, component.ErrNotFound) {
+ ctx.PureJSON(http.StatusNotFound, types.LFSLockError{
+ Message: "unable to delete lock : not found",
+ })
+ return
+ }
+ if errors.Is(err, component.ErrPermissionDenied) {
+ ctx.PureJSON(http.StatusMethodNotAllowed, types.LFSLockError{
+ Message: "unable to delete lock : this lock is not created by you, try --force if you still want unlock it",
+ })
+ return
+ }
+ slog.Error("Bad request format", "error", err)
+ ctx.PureJSON(http.StatusInternalServerError, types.LFSLockError{
+ Message: "unable to delete lock : Internal Server Error",
+ })
+ return
+ }
+ ctx.PureJSON(http.StatusOK, types.LFSLockResponse{
+ Lock: &types.LFSLock{
+ ID: strconv.FormatInt(lock.ID, 10),
+ Path: lock.Path,
+ LockedAt: lock.CreatedAt,
+ Owner: &types.LFSLockOwner{
+ Name: lock.User.Username,
+ },
+ },
+ })
+}
+
+func getService(r *http.Request) string {
+ if r.Method == "GET" {
+ return r.URL.Query().Get("service")
+ }
+ return filepath.Base(r.URL.Path)
+}
+
+type gzipResponseWriter struct {
+ gin.ResponseWriter
+ writer *gzip.Writer
+}
+
+func (g *gzipResponseWriter) Write(data []byte) (int, error) {
+ return g.writer.Write(data)
+}
+
+func (g *gzipResponseWriter) WriteString(s string) (int, error) {
+ return g.writer.Write([]byte(s))
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "net/http"
+ "net/url"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewHFDatasetHandler(config *config.Config) (*HFDatasetHandler, error) {
+ c, err := component.NewHFDatasetComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &HFDatasetHandler{
+ dc: c,
+ }, nil
+}
+
+type HFDatasetHandler struct {
+ dc component.HFDatasetComponent
+}
+
+func (h *HFDatasetHandler) DatasetPathsInfo(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format for dataset path info", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ref := ctx.Param("ref")
+
+ mappedBranch := ctx.Param("branch_mapped")
+ if mappedBranch != "" {
+ ref = mappedBranch
+ }
+
+ body, err := ctx.GetRawData()
+ // body: paths=test-00000-of-00001.parquet&expand=True
+ if err != nil {
+ httpbase.BadRequest(ctx, "failed to read request body for dataset path")
+ return
+ }
+ slog.Debug("Received body for dataset paths info", slog.Any("body", string(body)))
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ params, err := url.ParseQuery(string(body))
+ if err != nil {
+ slog.Error("error parsing query body", slog.Any("body", string(body)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ var req types.PathReq
+ req.Path = params.Get("paths")
+ req.Expand = params.Get("expand") == "True"
+ req.Namespace = namespace
+ req.Name = name
+ req.Ref = ref
+ req.CurrentUser = currentUser
+ slog.Debug("Received req for dataset paths info", slog.Any("req", req))
+
+ res, err := h.dc.GetPathsInfo(ctx, req)
+ if err != nil {
+ slog.Error("fail to get dataset paths info", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ }
+ ctx.PureJSON(http.StatusOK, res)
+}
+
+func (h *HFDatasetHandler) DatasetTree(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format for dataset tree", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ref := ctx.Param("ref")
+
+ mappedBranch := ctx.Param("branch_mapped")
+ if mappedBranch != "" {
+ ref = mappedBranch
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ pathInRepo := ctx.Param("path_in_repo")
+ slog.Debug("get path_in_repo in repo", slog.Any("pathInRepo", pathInRepo))
+
+ var req types.PathReq
+ req.Namespace = namespace
+ req.Name = name
+ req.Ref = ref
+ req.Path = pathInRepo
+ req.CurrentUser = currentUser
+ slog.Debug("received req for tree", slog.Any("req", req))
+
+ tree, err := h.dc.GetDatasetTree(ctx, req)
+ if err != nil {
+ slog.Error("fail to get dataset tree", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ }
+ ctx.PureJSON(http.StatusOK, tree)
+}
+
+func (h *HFDatasetHandler) HandleHFYaml(ctx *gin.Context) {
+ // request file: /api/datasets/haihui/ds1/resolve/7e6d54428819cb0f004fcc04e77a003a42c219b1/.huggingface.yaml
+ // HF return: {"error":"Sorry, we can't find the page you are looking for."}
+ response := types.HFErrorRes{
+ Error: "Sorry, we can't find the page you are looking for.",
+ }
+ ctx.PureJSON(http.StatusOK, response)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "net/http"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "go.temporal.io/sdk/client"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/api/workflow"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewInternalHandler(config *config.Config) (*InternalHandler, error) {
+ uc, err := component.NewInternalComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &InternalHandler{
+ c: uc,
+ config: config,
+ }, nil
+}
+
+type InternalHandler struct {
+ c component.InternalComponent
+ config *config.Config
+}
+
+// TODO: add prmission check
+func (h *InternalHandler) Allowed(ctx *gin.Context) {
+ allowed, err := h.c.Allowed(ctx)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.PureJSON(http.StatusOK, gin.H{
+ "status": allowed,
+ "message": "allowed",
+ })
+}
+
+func (h *InternalHandler) SSHAllowed(ctx *gin.Context) {
+ var (
+ req types.SSHAllowedReq
+ rawReq types.GitalyAllowedReq
+ repoPath string
+ )
+ if err := ctx.ShouldBind(&rawReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ if rawReq.Protocol == "ssh" {
+ if rawReq.GlRepository != "" {
+ repoPath = rawReq.GlRepository
+ } else {
+ repoPath = rawReq.Project
+ }
+ req.RepoType, req.Namespace, req.Name = getRepoInfoFronClonePath(repoPath)
+ req.Action = rawReq.Action
+ req.Changes = rawReq.Changes
+ req.KeyID = rawReq.KeyID
+ req.Protocol = rawReq.Protocol
+ req.CheckIP = rawReq.CheckIP
+
+ resp, err := h.c.SSHAllowed(ctx, req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ ctx.PureJSON(http.StatusOK, resp)
+ } else {
+ ctx.PureJSON(http.StatusOK, gin.H{
+ "status": true,
+ "message": "allowed",
+ })
+ }
+}
+
+// func (h *InternalHandler) PreReceive(ctx *gin.Context) {
+// body, _ := io.ReadAll(ctx.Request.Body)
+// strBody := string(body)
+// fmt.Printf(strBody)
+
+// var (
+// // req types.SSHAllowedReq
+// rawReq types.GitalyAllowedReq
+// )
+// if err := ctx.ShouldBind(&rawReq); err != nil {
+// slog.Error("Bad request format", "error", err)
+// httpbase.BadRequest(ctx, err.Error())
+// return
+// }
+// // req.RepoType, req.Namespace, req.Name = getRepoInfoFronClonePath(rawReq.GlRepository)
+
+// // resp, err := h.c.SSHAllowed(ctx, req)
+// // if err != nil {
+// // httpbase.ServerError(ctx, err)
+// // return
+// // }
+// ctx.PureJSON(http.StatusOK, gin.H{
+// "status": true,
+// "message": "allowed",
+// })
+// }
+
+func (h *InternalHandler) LfsAuthenticate(ctx *gin.Context) {
+ var req types.LfsAuthenticateReq
+ if err := ctx.ShouldBind(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.RepoType, req.Namespace, req.Name = getRepoInfoFronClonePath(req.Repo)
+ resp, err := h.c.LfsAuthenticate(ctx, req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.PureJSON(http.StatusOK, resp)
+}
+
+// TODO: add logic
+func (h *InternalHandler) PreReceive(ctx *gin.Context) {
+ ctx.PureJSON(http.StatusOK, gin.H{
+ "reference_counter_increased": true,
+ })
+}
+
+// TODO: add logic
+func (h *InternalHandler) PostReceive(ctx *gin.Context) {
+ var req types.PostReceiveReq
+ if err := ctx.ShouldBind(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ strs := strings.Split(req.Changes, " ")
+ // the format of originalRef is refs/heads/main
+ originalRef := strings.ReplaceAll(strs[2], "\n", "")
+ ref := strings.Split(strs[2], "/")[2]
+ // the format of ref is main
+ ref = strings.ReplaceAll(ref, "\n", "")
+ paths := strings.Split(req.GlRepository, "/")
+ diffReq := types.GetDiffBetweenTwoCommitsReq{
+ LeftCommitId: strs[0],
+ RightCommitId: strs[1],
+ Namespace: paths[1],
+ Name: paths[2],
+ Ref: ref,
+ RepoType: types.RepositoryType(strings.TrimSuffix(paths[0], "s")),
+ }
+ callback, err := h.c.GetCommitDiff(ctx, diffReq)
+ if err != nil {
+ slog.Error("post receive: failed to get commit diff", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ callback.Ref = originalRef
+
+ //start workflow to handle push request
+ workflowClient := workflow.GetWorkflowClient()
+ workflowOptions := client.StartWorkflowOptions{
+ TaskQueue: workflow.HandlePushQueueName,
+ }
+
+ we, err := workflowClient.ExecuteWorkflow(ctx, workflowOptions, workflow.HandlePushWorkflow,
+ callback,
+ h.config,
+ )
+ if err != nil {
+ slog.Error("failed to handle git push callback", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("start handle push workflow", slog.String("workflow_id", we.GetID()), slog.Any("req", callback))
+
+ ctx.PureJSON(http.StatusOK, gin.H{
+ "reference_counter_decreased": true,
+ "messages": []Messages{
+ {
+ Message: "Welcome to OpenCSG!",
+ Type: "alert",
+ },
+ },
+ })
+}
+
+func (h *InternalHandler) GetAuthorizedKeys(ctx *gin.Context) {
+ key := ctx.Query("key")
+ sshKey, err := h.c.GetAuthorizedKeys(ctx, key)
+ if err != nil {
+ slog.Error("failed to get authorize keys", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.PureJSON(http.StatusOK, gin.H{
+ "id": sshKey.ID,
+ "key": sshKey.Content,
+ })
+}
+
+type Messages struct {
+ Message string `json:"message"`
+ Type string `json:"type"`
+}
+
+func getRepoInfoFronClonePath(clonePath string) (repoType types.RepositoryType, namespace, name string) {
+ repoWithoutSuffix := strings.TrimSuffix(clonePath, ".git")
+ repoWithoutPrefix := strings.TrimPrefix(repoWithoutSuffix, "/")
+ paths := strings.Split(repoWithoutPrefix, "/")
+ repoType = types.RepositoryType(strings.TrimSuffix(paths[0], "s"))
+ namespace = paths[1]
+ name = paths[2]
+ return
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/builder/proxy"
+)
+
+type InternalServiceProxyHandler struct {
+ rp *proxy.ReverseProxy
+}
+
+func NewInternalServiceProxyHandler(remoteEndpoint string) (*InternalServiceProxyHandler, error) {
+ rp, err := proxy.NewReverseProxy(remoteEndpoint)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create reverse proxy: %w", err)
+ }
+ return &InternalServiceProxyHandler{rp: rp}, nil
+}
+
+// Proxy send request to backend service, without change the request path
+func (h *InternalServiceProxyHandler) Proxy(ctx *gin.Context) {
+ // Log the request URL and header
+ slog.Debug("http request", slog.Any("request", ctx.Request.URL), slog.Any("header", ctx.Request.Header))
+
+ // Serve the request using the router
+ h.rp.ServeHTTP(ctx.Writer, ctx.Request, "")
+}
+
+// ProxyToApi similar with Proxy, but change the request path
+//
+// the target request path can read params from the origin request path. This request will read 'username' from original request:
+//
+// apiGroup.PUT("/users/:username", userProxyHandler.ProxyToApi("/api/v1/user/%v", "username"))
+func (h *InternalServiceProxyHandler) ProxyToApi(api string, originParams ...string) gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ slog.Info("proxy user request", slog.Any("request", ctx.Request.URL), slog.Any("header", ctx.Request.Header))
+ finalApi := api
+ if len(originParams) > 0 {
+ var params []any
+ for _, op := range originParams {
+ params = append(params, ctx.Param(op))
+ }
+ finalApi = fmt.Sprintf(finalApi, params...)
+ }
+ h.rp.ServeHTTP(ctx.Writer, ctx.Request, finalApi)
+ }
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+type LicenseHandler struct {
+ lc component.LicenseComponent
+}
+
+func NewLicenseHandler(config *config.Config) (*LicenseHandler, error) {
+ lc, err := component.NewLicenseComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create license component, err: %w", err)
+ }
+ return &LicenseHandler{lc: lc}, nil
+}
+
+// ListLicense godoc
+// @Security ApiKey
+// @Summary List license
+// @Description List license
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param product query string false "product"
+// @Param edition query string false "edition"
+// @Param search query string false "search"
+// @Param current_user query string true "current user"
+// @Success 200 {object} types.Response{data=[]database.License,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses [get]
+func (h *LicenseHandler) ListLicenses(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format prompt list", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ product := ctx.Query("product")
+ edition := ctx.Query("edition")
+ search := ctx.Query("search")
+
+ req := types.QueryLicenseReq{
+ Product: product,
+ Edition: edition,
+ Search: search,
+ CurrentUser: currentUser,
+ Per: per,
+ Page: page,
+ }
+
+ licenses, total, err := h.lc.ListLicense(ctx, req)
+ if err != nil {
+ slog.Error("Failed to list licenses", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": licenses,
+ "total": total,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+// CreateLicense godoc
+// @Security ApiKey
+// @Summary Create a license
+// @Description Create a license
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current user"
+// @Param body body types.CreateLicenseReq true "body"
+// @Success 200 {object} types.Response{string} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses [post]
+func (h *LicenseHandler) CreateLicense(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ var req *types.CreateLicenseReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad license request body format", "error", err)
+ httpbase.BadRequest(ctx, fmt.Sprintf("Bad license request body format, error: %v", err))
+ return
+ }
+ req.CurrentUser = currentUser
+ req.Key = ""
+ license, err := h.lc.CreateLicense(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create license", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, license)
+}
+
+// GetLicense godoc
+// @Security ApiKey
+// @Summary Get a license by id
+// @Description Get a license by id
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Param create_license_file query bool false "create license file" default(false)
+// @Param current_user query string true "current user"
+// @Success 200 {object} types.Response{data=database.License,license=string} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses/{id} [get]
+func (h *LicenseHandler) GetLicense(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ createFileStr := ctx.Query("create_license_file")
+ createFile, err := strconv.ParseBool(createFileStr)
+ if err != nil {
+ createFile = false
+ }
+
+ req := types.GetLicenseReq{
+ ID: id,
+ CreateFile: createFile,
+ CurrentUser: currentUser,
+ }
+ license, fileContent, err := h.lc.GetLicenseByID(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get license", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "license": fileContent,
+ "data": license,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+// UpdateLicense godoc
+// @Security ApiKey
+// @Summary Update a license by id
+// @Description Update a license by id
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Param current_user query string true "current user"
+// @Param body body types.UpdateLicenseReq true "body"
+// @Success 200 {object} types.Response{database.License} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses/{id} [put]
+func (h *LicenseHandler) UpdateLicense(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdateLicenseReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad license request body format", "error", err)
+ httpbase.BadRequest(ctx, fmt.Sprintf("Bad license request body format, error: %v", err))
+ return
+ }
+
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.CurrentUser = currentUser
+ license, err := h.lc.UpdateLicense(ctx, id, req)
+ if err != nil {
+ slog.Error("Failed to update license", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, license)
+}
+
+// DeleteLicense godoc
+// @Security ApiKey
+// @Summary Delete a license by id
+// @Description Delete a license by id
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Param current_user query string true "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses/{id} [delete]
+func (h *LicenseHandler) DeleteLicense(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = h.lc.DeleteLicenseByID(ctx, id, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete license", slog.Any("error", err), slog.Int64("id", id))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// ImportLicense godoc
+// @Security ApiKey
+// @Summary Import a license
+// @Description Import a license
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current user"
+// @Param body body types.ImportLicenseReq true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses/license [put]
+func (h *LicenseHandler) ImportLicense(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.ImportLicenseReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad import license request body format", "error", err)
+ httpbase.BadRequest(ctx, fmt.Sprintf("Bad import license request body format, error: %v", err))
+ return
+ }
+ req.CurrentUser = currentUser
+ err := h.lc.ImportLicense(ctx, req)
+ if err != nil {
+ slog.Error("Failed to import license", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetLicenseStatus godoc
+// @Security ApiKey
+// @Summary Get active license status
+// @Description Get active license status
+// @Tags License
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{types.LicenseStatusResp} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses/license [get]
+func (h *LicenseHandler) GetLicenseStatus(ctx *gin.Context) {
+ status, err := h.lc.GetLicenseStatus(ctx)
+ if err != nil {
+ slog.Error("Failed to get license status", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, status)
+}
+
+// VerifyLicense godoc
+// @Security ApiKey
+// @Summary Verify a license
+// @Description Verify a license
+// @Tags License
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current user"
+// @Param body body types.ImportLicenseReq true "body"
+// @Success 200 {object} types.Response{types.RSAPayload} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /licenses/license [post]
+func (h *LicenseHandler) VerifyLicense(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.ImportLicenseReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad verify license request body format", "error", err)
+ httpbase.BadRequest(ctx, fmt.Sprintf("Bad import license request body format, error: %v", err))
+ return
+ }
+ req.CurrentUser = currentUser
+ licInfo, err := h.lc.VerifyLicense(ctx, req)
+ if err != nil {
+ slog.Error("Failed to verify license", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, licInfo)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewListHandler(config *config.Config) (*ListHandler, error) {
+ uc, err := component.NewListComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSpaceComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create space component,%w", err)
+ }
+ return &ListHandler{
+ c: uc,
+ sc: sc,
+ }, nil
+}
+
+type ListHandler struct {
+ c component.ListComponent
+ sc component.SpaceComponent
+}
+
+// ListTrendingModels godoc
+// @Security ApiKey
+// @Summary List models by paths
+// @Description list models by paths
+// @Tags List
+// @Accept json
+// @Produce json
+// @Param body body types.ListByPathReq true "body"
+// @Success 200 {object} types.Response{data=[]types.ModelResp} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /list/models_by_path [post]
+func (h *ListHandler) ListModelsByPath(ctx *gin.Context) {
+ var listTrendingReq types.ListByPathReq
+ if err := ctx.ShouldBindJSON(&listTrendingReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ resp, err := h.c.ListModelsByPath(ctx, &listTrendingReq)
+ if err != nil {
+ slog.Error("Failed to update dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// ListTrendingDatasets godoc
+// @Security ApiKey
+// @Summary List datasets by paths
+// @Description list datasets by paths
+// @Tags List
+// @Accept json
+// @Produce json
+// @Param body body types.ListByPathReq true "body"
+// @Success 200 {object} types.Response{data=[]types.DatasetResp} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /list/datasets_by_path [post]
+func (h *ListHandler) ListDatasetsByPath(ctx *gin.Context) {
+ var listTrendingReq types.ListByPathReq
+ if err := ctx.ShouldBindJSON(&listTrendingReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ resp, err := h.c.ListDatasetsByPath(ctx, &listTrendingReq)
+ if err != nil {
+ slog.Error("Failed to update dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// ListTrendingSpaces godoc
+// @Security ApiKey
+// @Summary List spaces by paths
+// @Tags List
+// @Accept json
+// @Produce json
+// @Param body body types.ListByPathReq true "body"
+// @Success 200 {object} types.Response{data=[]types.Space} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /list/spaces_by_path [post]
+func (h *ListHandler) ListSpacesByPath(ctx *gin.Context) {
+ var listTrendingReq types.ListByPathReq
+ if err := ctx.ShouldBindJSON(&listTrendingReq); err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ resp, err := h.sc.ListByPath(ctx, listTrendingReq.Paths)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+// create new MirrorHandler
+func NewMirrorHandler(config *config.Config) (*MirrorHandler, error) {
+ mc, err := component.NewMirrorComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &MirrorHandler{
+ mc: mc,
+ }, nil
+}
+
+type MirrorHandler struct {
+ mc component.MirrorComponent
+}
+
+// CreateMirrorRepo godoc
+// @Security ApiKey
+// @Summary Create mirror repo
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param body body types.CreateMirrorRepoReq true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/repo [post]
+func (h *MirrorHandler) CreateMirrorRepo(ctx *gin.Context) {
+ var req types.CreateMirrorRepoReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ req.CurrentUser = currentUser
+ m, err := h.mc.CreateMirrorRepo(ctx, req)
+ if err != nil {
+ slog.Error("failed to create mirror repo", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Debug("create mirror repo", slog.Any("mirror", m.Repository), slog.Any("req", req))
+
+ httpbase.OK(ctx, nil)
+}
+
+// GetMirrorRepos godoc
+// @Security ApiKey
+// @Summary Get mirror repos
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param per query int false "per" default(20)
+// @Param page query int false "page" default(1)
+// @Success 200 {object} types.Response{data=[]types.MirrorRepo,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/repos [get]
+func (h *MirrorHandler) Repos(ctx *gin.Context) {
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ repos, total, err := h.mc.Repos(ctx, currentUser, per, page)
+ if err != nil {
+ slog.Error("failed to get mirror repos", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": repos,
+ "total": total,
+ }
+
+ httpbase.OK(ctx, respData)
+}
+
+// GetMirrors godoc
+// @Security ApiKey
+// @Summary Get mirrors
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param per query int false "per" default(20)
+// @Param page query int false "page" default(1)
+// @Success 200 {object} types.Response{data=[]types.Mirror,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirrors [get]
+func (h *MirrorHandler) Index(ctx *gin.Context) {
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ search := ctx.Query("search")
+ repos, total, err := h.mc.Index(ctx, currentUser, per, page, search)
+ if err != nil {
+ slog.Error("failed to get mirror repos", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": repos,
+ "total": total,
+ }
+
+ httpbase.OK(ctx, respData)
+}
+
+// GetMirrorStatusCounts godoc
+// @Security ApiKey
+// @Summary Get mirror status counts
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{data=[]types.MirrorStatusCount} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/statistics [get]
+func (h *MirrorHandler) Statistics(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ statusCounts, err := h.mc.Statistics(ctx, currentUser)
+ if err != nil {
+ slog.Error("failed to get mirror statistics", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, statusCounts)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewMirrorSourceHandler(config *config.Config) (*MirrorSourceHandler, error) {
+ c, err := component.NewMirrorSourceComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &MirrorSourceHandler{
+ c: c,
+ }, nil
+}
+
+type MirrorSourceHandler struct {
+ c component.MirrorSourceComponent
+}
+
+// CreateMirrorSource godoc
+// @Security ApiKey
+// @Summary Create mirror source
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param body body types.CreateMirrorSourceReq true "body"
+// @Success 200 {object} types.Response{data=database.MirrorSource} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/sources [post]
+func (h *MirrorSourceHandler) Create(ctx *gin.Context) {
+ var msReq types.CreateMirrorSourceReq
+ if err := ctx.ShouldBindJSON(&msReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ msReq.CurrentUser = currentUser
+ ms, err := h.c.Create(ctx, msReq)
+ if err != nil {
+ slog.Error("Failed to create mirror source", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ms)
+}
+
+// GetMirrorSources godoc
+// @Security ApiKey
+// @Summary Get mirror sources
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{data=[]database.MirrorSource} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/sources [get]
+func (h *MirrorSourceHandler) Index(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ ms, err := h.c.Index(ctx, currentUser)
+ if err != nil {
+ slog.Error("Failed to get mirror sources", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ms)
+}
+
+// UpdateMirrorSource godoc
+// @Security ApiKey
+// @Summary Update mirror source
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Param body body types.UpdateMirrorSourceReq true "body"
+// @Success 200 {object} types.Response{data=database.MirrorSource} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/sources/{id} [put]
+func (h *MirrorSourceHandler) Update(ctx *gin.Context) {
+ var msReq types.UpdateMirrorSourceReq
+ var msId int64
+ id := ctx.Param("id")
+ if id == "" {
+ err := fmt.Errorf("invalid mirror source id")
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ if err := ctx.ShouldBindJSON(&msReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ msId, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ msReq.ID = msId
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ msReq.CurrentUser = currentUser
+ ms, err := h.c.Update(ctx, msReq)
+ if err != nil {
+ slog.Error("Failed to get mirror sources", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ms)
+}
+
+// GetMirrorSource godoc
+// @Security ApiKey
+// @Summary Get mirror source
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{data=database.MirrorSource} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/sources/{id} [get]
+func (h *MirrorSourceHandler) Get(ctx *gin.Context) {
+ var msId int64
+ id := ctx.Param("id")
+ if id == "" {
+ err := fmt.Errorf("invalid mirror source id")
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ msId, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ ms, err := h.c.Get(ctx, msId, currentUser)
+ if err != nil {
+ slog.Error("Failed to get mirror source", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ms)
+}
+
+// DeleteMirrorSource godoc
+// @Security ApiKey
+// @Summary Delete mirror source
+// @Tags Mirror
+// @Accept json
+// @Produce json
+// @Param id path string true "id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /mirror/sources/{id} [delete]
+func (h *MirrorSourceHandler) Delete(ctx *gin.Context) {
+ var msId int64
+ id := ctx.Param("id")
+ if id == "" {
+ err := fmt.Errorf("invalid mirror source id")
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ msId, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ err = h.c.Delete(ctx, msId, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete mirror source", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "slices"
+ "strconv"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewModelHandler(config *config.Config) (*ModelHandler, error) {
+ uc, err := component.NewModelComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ repo, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating repo component:%w", err)
+ }
+ return &ModelHandler{
+ c: uc,
+ sc: sc,
+ repo: repo,
+ }, nil
+}
+
+type ModelHandler struct {
+ c component.ModelComponent
+ repo component.RepoComponent
+ sc component.SensitiveComponent
+}
+
+// GetVisiableModels godoc
+// @Security ApiKey
+// @Summary Get Visiable models for current user
+// @Description get visiable models for current user
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param search query string false "search text"
+// @Param task_tag query string false "filter by task tag, deprecated"
+// @Param framework_tag query string false "filter by framework tag, deprecated"
+// @Param license_tag query string false "filter by license tag, deprecated"
+// @Param language_tag query string false "filter by language tag, deprecated"
+// @Param tag_category query string false "filter by tag category"
+// @Param tag_name query string false "filter by tag name"
+// @Param need_op_weight query bool false "need op weight" default(false)
+// @Param sort query string false "sort by"
+// @Param source query string false "source" Enums(opencsg, huggingface, local)
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Model,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models [get]
+func (h *ModelHandler) Index(ctx *gin.Context) {
+ filter := new(types.RepoFilter)
+ filter.Tags = parseTagReqs(ctx)
+ filter.Username = httpbase.GetCurrentUser(ctx)
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filter = getFilterFromContext(ctx, filter)
+ if !slices.Contains(Sorts, filter.Sort) {
+ msg := fmt.Sprintf("sort parameter must be one of %v", Sorts)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ if filter.Source != "" && !slices.Contains[[]string](Sources, filter.Source) {
+ msg := fmt.Sprintf("source parameter must be one of %v", Sources)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ qNeedOpWeight := ctx.Query("need_op_weight")
+ needOpWeight, err := strconv.ParseBool(qNeedOpWeight)
+ if err != nil {
+ needOpWeight = false
+ }
+ models, total, err := h.c.Index(ctx, filter, per, page, needOpWeight)
+ if err != nil {
+ slog.Error("Failed to get models", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get public models succeed", slog.Int("count", total))
+ respData := gin.H{
+ "data": models,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// CreateModel godoc
+// @Security ApiKey
+// @Summary Create a new model
+// @Description create a new model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param body body types.CreateModelReq true "body"
+// @Success 200 {object} types.Response{data=database.Model} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models [post]
+func (h *ModelHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.CreateModelReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Username = currentUser
+
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ model, err := h.c.Create(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create model", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create model succeed", slog.String("model", model.Name))
+ httpbase.OK(ctx, model)
+}
+
+// UpdateModel godoc
+// @Security ApiKey
+// @Summary Update a exists model
+// @Description update a exists model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the model owner"
+// @Param body body types.UpdateModelReq true "body"
+// @Success 200 {object} types.Response{data=database.Model} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name} [put]
+func (h *ModelHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdateModelReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.Username = currentUser
+
+ model, err := h.c.Update(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update model", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Update model succeed", slog.String("model", model.Name))
+ httpbase.OK(ctx, model)
+}
+
+// DeleteModel godoc
+// @Security ApiKey
+// @Summary Delete a exists model
+// @Description delete a exists model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the model owner"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name} [delete]
+func (h *ModelHandler) Delete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.Delete(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete model", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Delete model succeed", slog.String("model", name))
+ httpbase.OK(ctx, nil)
+}
+
+// GetModel godoc
+// @Security ApiKey
+// @Summary Get model detail
+// @Description get model detail
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Param need_op_weight query bool false "need op weight" default(false)
+// @Success 200 {object} types.Response{data=types.Model} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name} [get]
+func (h *ModelHandler) Show(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ qNeedOpWeight := ctx.Query("need_op_weight")
+ needOpWeight, err := strconv.ParseBool(qNeedOpWeight)
+ if err != nil {
+ needOpWeight = false
+ }
+ detail, err := h.c.Show(ctx, namespace, name, currentUser, needOpWeight)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get model detail", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get model succeed", slog.String("model", name))
+ httpbase.OK(ctx, detail)
+}
+
+func (h *ModelHandler) SDKModelInfo(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ref := ctx.Param("ref")
+ mappedBranch := ctx.Param("branch_mapped")
+ if mappedBranch != "" {
+ ref = mappedBranch
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ modelInfo, err := h.c.SDKModelInfo(ctx, namespace, name, ref, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get sdk model info", slog.String("namespace", namespace), slog.String("name", name), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ ctx.JSON(http.StatusOK, modelInfo)
+}
+
+// ModelRelations godoc
+// @Security ApiKey
+// @Summary Get model related assets
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{data=types.Relations} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/relations [get]
+func (h *ModelHandler) Relations(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.c.Relations(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get model relations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+// SetRelation godoc
+// @Security ApiKey
+// @Summary Set dataset relation for model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param req body types.RelationDatasets true "set dataset relation"
+// @Success 200 {object} types.Response{data=types.Relations} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/relations [put]
+func (h *ModelHandler) SetRelations(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RelationDatasets
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.CurrentUser = currentUser
+
+ err = h.c.SetRelationDatasets(ctx, req)
+ if err != nil {
+ slog.Error("Failed to set datasets for model", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// AddDatasetRelation godoc
+// @Security ApiKey
+// @Summary add dataset relation for model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param req body types.RelationDataset true "add dataset relation"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/relations/dataset [post]
+func (h *ModelHandler) AddDatasetRelation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RelationDataset
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.CurrentUser = currentUser
+
+ err = h.c.AddRelationDataset(ctx, req)
+ if err != nil {
+ slog.Error("Failed to add dataset for model", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteDatasetRelation godoc
+// @Security ApiKey
+// @Summary delete dataset relation for model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param req body types.RelationDataset true "delelet dataset relation"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/relations/dataset [delete]
+func (h *ModelHandler) DelDatasetRelation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RelationDataset
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.CurrentUser = currentUser
+
+ err = h.c.DelRelationDataset(ctx, req)
+ if err != nil {
+ slog.Error("Failed to delete dataset for model", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+func parseTagReqs(ctx *gin.Context) (tags []types.TagReq) {
+ tagCategories := ctx.QueryArray("tag_category")
+ tagNames := ctx.QueryArray("tag_name")
+ if len(tagCategories) > 0 && len(tagCategories) == len(tagNames) {
+ for i, category := range tagCategories {
+ tags = append(tags, types.TagReq{
+ Name: tagNames[i],
+ Category: category,
+ })
+ }
+ return
+ }
+
+ licenseTag := ctx.Query("license_tag")
+ taskTag := ctx.Query("task_tag")
+ frameworkTag := ctx.Query("framework_tag")
+ if licenseTag != "" {
+ tags = append(tags, types.TagReq{
+ Name: strings.ToLower(licenseTag),
+ Category: "license",
+ })
+ }
+
+ if taskTag != "" {
+ tags = append(tags, types.TagReq{
+ Name: strings.ToLower(taskTag),
+ Category: "task",
+ })
+ }
+
+ if frameworkTag != "" {
+ tags = append(tags, types.TagReq{
+ Name: strings.ToLower(frameworkTag),
+ Category: "framework",
+ })
+ }
+
+ languageTag := ctx.Query("language_tag")
+ if languageTag != "" {
+ tags = append(tags, types.TagReq{
+ Name: strings.ToLower(languageTag),
+ Category: "language",
+ })
+ }
+
+ industryTag := ctx.Query("industry_tag")
+ if industryTag != "" {
+ tags = append(tags, types.TagReq{
+ Name: strings.ToLower(industryTag),
+ Category: "industry",
+ })
+ }
+ return
+}
+
+func convertFilePathFromRoute(path string) string {
+ return strings.TrimLeft(path, "/")
+}
+
+// ModelRun godoc
+// @Security ApiKey
+// @Summary run model as inference
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Param body body types.ModelRunReq true "deploy setting of inference"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/run [post]
+func (h *ModelHandler) DeployDedicated(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ allow, err := h.repo.AllowReadAccess(ctx, types.ModelRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+ if !allow {
+ slog.Info("user not allowed to run model", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ httpbase.UnauthorizedError(ctx, errors.New("user not allowed to run model"))
+ return
+ }
+
+ var req types.ModelRunReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if req.Revision == "" {
+ req.Revision = "main" // default repo branch
+ }
+
+ if req.MinReplica < 0 || req.MaxReplica < 0 || req.MinReplica > req.MaxReplica {
+ slog.Error("Bad request setting for replica", slog.Any("MinReplica", req.MinReplica), slog.Any("MaxReplica", req.MaxReplica))
+ httpbase.BadRequest(ctx, "Bad request setting for replica")
+ return
+ }
+ // for reserved resource,no scaling
+ if req.OrderDetailID != 0 {
+ req.MinReplica = 1
+ req.MaxReplica = 1
+ }
+
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ epReq := types.DeployActReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployType: types.InferenceType,
+ }
+ deployID, err := h.c.Deploy(ctx, epReq, req)
+ if err != nil {
+ slog.Error("failed to deploy model as inference", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("currentUser", currentUser), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Debug("deploy model as inference created", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Int64("deploy_id", deployID))
+
+ // return deploy_id
+ response := types.DeployRepo{DeployID: deployID}
+
+ httpbase.OK(ctx, response)
+}
+
+// FinetuneCreate godoc
+// @Security ApiKey
+// @Summary create a finetune instance
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Param body body types.InstanceRunReq true "deploy setting of instance"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/finetune [post]
+func (h *ModelHandler) FinetuneCreate(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ allow, err := h.repo.AllowAdminAccess(ctx, types.ModelRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+ if !allow {
+ slog.Info("user is not allowed to run model", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ httpbase.UnauthorizedError(ctx, errors.New("user not allowed to run model"))
+ return
+ }
+
+ var req types.InstanceRunReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ modelReq := &types.ModelRunReq{
+ DeployName: req.DeployName,
+ ClusterID: req.ClusterID,
+ ResourceID: req.ResourceID,
+ RuntimeFrameworkID: req.RuntimeFrameworkID,
+ MinReplica: 1,
+ MaxReplica: 1,
+ SecureLevel: 2,
+ Revision: req.Revision,
+ OrderDetailID: req.OrderDetailID,
+ }
+
+ ftReq := types.DeployActReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployType: types.FinetuneType,
+ }
+
+ deployID, err := h.c.Deploy(ctx, ftReq, *modelReq)
+ if err != nil {
+ slog.Error("failed to deploy model as notebook instance", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Debug("deploy model as instance created", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Int64("deploy_id", deployID))
+
+ // return deploy_id
+ response := types.DeployRepo{DeployID: deployID}
+
+ httpbase.OK(ctx, response)
+}
+
+// DeleteDeploy godoc
+// @Security ApiKey
+// @Summary Delete a model inference
+// @Description delete a model inference
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/run/{id} [delete]
+func (h *ModelHandler) DeployDelete(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ delReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.InferenceType,
+ }
+ err = h.repo.DeleteDeploy(ctx, delReq)
+ if err != nil {
+ slog.Error("Failed to delete deploy", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// FinetuneDelete godoc
+// @Security ApiKey
+// @Summary Delete a finetune instance
+// @Description delete a finetune instance
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/finetune/{id} [delete]
+func (h *ModelHandler) FinetuneDelete(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ delReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.FinetuneType,
+ }
+ err = h.repo.DeleteDeploy(ctx, delReq)
+ if err != nil {
+ slog.Error("Failed to delete deploy", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// StopDeploy godoc
+// @Security ApiKey
+// @Summary Stop a model inference
+// @Description Stop a model inference
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/run/{id}/stop [put]
+func (h *ModelHandler) DeployStop(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ stopReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.InferenceType,
+ }
+ err = h.repo.DeployStop(ctx, stopReq)
+ if err != nil {
+ slog.Error("Failed to stop deploy", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// StartDeploy godoc
+// @Security ApiKey
+// @Summary Start a model inference
+// @Description Start a model inference
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "deploy id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/run/{id}/start [put]
+func (h *ModelHandler) DeployStart(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ startReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.InferenceType,
+ }
+
+ err = h.repo.DeployStart(ctx, startReq)
+ if err != nil {
+ slog.Error("Failed to start deploy", slog.Any("error", err), slog.Any("repoType", types.ModelRepo), slog.String("namespace", namespace), slog.String("name", name), slog.Any("deployID", id))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetModelsByRuntime godoc
+// @Security ApiKey
+// @Summary Get Visible models by runtime framework for current user
+// @Description get visible models by runtime framework for current user
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Param current_user query string false "current user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2) default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id}/models [get]
+func (h *ModelHandler) ListByRuntimeFrameworkID(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ models, total, err := h.c.ListModelsByRuntimeFrameworkID(ctx, currentUser, per, page, id, deployType)
+ if err != nil {
+ slog.Error("Failed to get models", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": models,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// FinetuneStop godoc
+// @Security ApiKey
+// @Summary Stop a finetune instance
+// @Description Stop a finetune instance
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/finetune/{id}/stop [put]
+func (h *ModelHandler) FinetuneStop(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ stopReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.FinetuneType,
+ }
+ err = h.repo.DeployStop(ctx, stopReq)
+ if err != nil {
+ slog.Error("Failed to stop deploy", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// FinetuneStart godoc
+// @Security ApiKey
+// @Summary Start a finetune instance
+// @Description Start a finetune instance
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "deploy id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/finetune/{id}/start [put]
+func (h *ModelHandler) FinetuneStart(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ startReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.FinetuneType,
+ }
+ err = h.repo.DeployStart(ctx, startReq)
+ if err != nil {
+ slog.Error("Failed to start deploy", slog.Any("error", err), slog.Any("repoType", types.ModelRepo), slog.String("namespace", namespace), slog.String("name", name), slog.Any("deployID", id))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetRuntime godoc
+// @Security ApiKey
+// @Summary Get all runtime frameworks for current user
+// @Description get all runtime frameworks for current user
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework [get]
+func (h *ModelHandler) ListAllRuntimeFramework(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ runtimes, err := h.c.ListAllByRuntimeFramework(ctx, currentUser)
+ if err != nil {
+ slog.Error("Failed to get runtime frameworks", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": runtimes,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// UpdateModelRuntime godoc
+// @Security ApiKey
+// @Summary Set model runtime frameworks
+// @Description set model runtime frameworks
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2) default(1)
+// @Param current_user query string false "current user"
+// @Param body body types.RuntimeFrameworkModels true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id} [post]
+func (h *ModelHandler) UpdateModelRuntimeFrameworks(ctx *gin.Context) {
+ var req types.RuntimeFrameworkModels
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ slog.Info("update runtime frameworks models", slog.Any("req", req), slog.Any("runtime framework id", id), slog.Any("deployType", deployType))
+
+ list, err := h.c.SetRuntimeFrameworkModes(ctx, deployType, id, req.Models)
+ if err != nil {
+ slog.Error("Failed to set models runtime framework", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, list)
+}
+
+// DeleteModelRuntime godoc
+// @Security ApiKey
+// @Summary Set model runtime frameworks
+// @Description set model runtime frameworks
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2) default(1)
+// @Param body body types.RuntimeFrameworkModels true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id} [delete]
+func (h *ModelHandler) DeleteModelRuntimeFrameworks(ctx *gin.Context) {
+ var req types.RuntimeFrameworkModels
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ slog.Info("update runtime frameworks models", slog.Any("req", req), slog.Any("runtime framework id", id), slog.Any("deployType", deployType))
+
+ list, err := h.c.DeleteRuntimeFrameworkModes(ctx, deployType, id, req.Models)
+ if err != nil {
+ slog.Error("Failed to set models runtime framework", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, list)
+}
+
+// GetRuntimeFrameworkModels godoc
+// @Security ApiKey
+// @Summary Get Visible models for all runtime frameworks for current user
+// @Description get visible models for all runtime frameworks for current user
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param search query string false "search text"
+// @Param sort query string false "sort by"
+// @Param current_user query string false "current user"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Param deploy_type query int false "deploy_type" Enums(1, 2) default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/models [get]
+func (h *ModelHandler) ListModelsOfRuntimeFrameworks(ctx *gin.Context) {
+ filter := new(types.RepoFilter)
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ filter = getFilterFromContext(ctx, filter)
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request deploy type format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request per and page format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ models, total, err := h.c.ListModelsOfRuntimeFrameworks(ctx, currentUser, filter.Search, filter.Sort, per, page, deployType)
+ if err != nil {
+ slog.Error("fail to get models for all runtime frameworks", slog.Any("deployType", deployType), slog.Any("per", per), slog.Any("page", page), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": models,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// ModelFiles godoc
+// @Security ApiKey
+// @Summary Get all files of a model
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{data=types.File} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/all_files [get]
+func (h *ModelHandler) AllFiles(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req types.GetAllFilesReq
+ req.Namespace = namespace
+ req.Name = name
+ req.RepoType = types.ModelRepo
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ detail, err := h.repo.AllFiles(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get model all files", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+// ModelServerless godoc
+// @Security ApiKey
+// @Summary run model as serverless service
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Param body body types.ModelRunReq true "deploy setting of serverless"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless [post]
+func (h *ModelHandler) DeployServerless(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.ModelRunReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if req.Revision == "" {
+ req.Revision = "main" // default repo branch
+ }
+
+ if req.MinReplica < 0 || req.MaxReplica < 0 || req.MinReplica > req.MaxReplica {
+ slog.Error("Bad request setting for replica", slog.Any("MinReplica", req.MinReplica), slog.Any("MaxReplica", req.MaxReplica))
+ httpbase.BadRequest(ctx, "Bad request setting for replica")
+ return
+ }
+
+ deployReq := types.DeployActReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployType: types.ServerlessType,
+ }
+
+ req.SecureLevel = 1 // public for serverless
+ deployID, err := h.c.Deploy(ctx, deployReq, req)
+ if err != nil {
+ slog.Error("failed to deploy model as serverless", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("currentUser", currentUser), slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Debug("deploy model as serverless created", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Int64("deploy_id", deployID))
+
+ // return deploy_id
+ response := types.DeployRepo{DeployID: deployID}
+
+ httpbase.OK(ctx, response)
+}
+
+// StartServerless godoc
+// @Security ApiKey
+// @Summary Start a model serverless
+// @Description Start a model serverless
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "deploy id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless/{id}/start [put]
+func (h *ModelHandler) ServerlessStart(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ startReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.ServerlessType,
+ }
+
+ err = h.repo.DeployStart(ctx, startReq)
+ if err != nil {
+ slog.Error("Failed to start deploy", slog.Any("error", err), slog.Any("repoType", types.ModelRepo), slog.String("namespace", namespace), slog.String("name", name), slog.Any("deployID", id))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// StopServerless godoc
+// @Security ApiKey
+// @Summary Stop a model serverless
+// @Description Stop a model serverless
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless/{id}/stop [put]
+func (h *ModelHandler) ServerlessStop(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ stopReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: id,
+ DeployType: types.ServerlessType,
+ }
+
+ err = h.repo.DeployStop(ctx, stopReq)
+ if err != nil {
+ slog.Error("Failed to stop deploy", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetServerless godoc
+// @Security JWT token
+// @Summary get model serverless
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless [get]
+func (h *ModelHandler) GetDeployServerless(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ response, err := h.c.GetServerless(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to get model serverless endpoint", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("currentUser", currentUser), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, response)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+type SyncHandler struct {
+ c component.MultiSyncComponent
+}
+
+func NewSyncHandler(config *config.Config) (*SyncHandler, error) {
+ c, err := component.NewMultiSyncComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create multi sync component: %w", err)
+ }
+ return &SyncHandler{
+ c: c,
+ }, nil
+}
+
+// Latest
+// @Security ApiKey
+// @Summary Get latest version
+// @Tags Sync
+// @Produce json
+// @Param cur query string true "current version"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{data=types.SyncVersionResponse} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /sync/version/latest [get]
+func (h *SyncHandler) Latest(c *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(c)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(c, component.ErrUserNotFound)
+ return
+ }
+
+ varCur := c.Query("cur")
+ cur, err := strconv.ParseInt(varCur, 10, 64)
+
+ if err != nil {
+ httpbase.BadRequest(c, fmt.Sprintf("invalid param cur: %s", err.Error()))
+ return
+ }
+ const limit int64 = 100
+ versions, err := h.c.More(c, cur, limit)
+ if err != nil {
+ httpbase.ServerError(c, fmt.Errorf("failed to get more versions: %w", err))
+ return
+ }
+
+ var resp types.SyncVersionData
+ resp.Versions = versions
+ if len(versions) == int(limit) {
+ resp.HasMore = true
+ }
+ httpbase.OK(c, resp)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewOrganizationHandler(config *config.Config) (*OrganizationHandler, error) {
+ sc, err := component.NewSpaceComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ cc, err := component.NewCodeComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ mc, err := component.NewModelComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ dsc, err := component.NewDatasetComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ colc, err := component.NewCollectionComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ pc, err := component.NewPromptComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &OrganizationHandler{
+ sc: sc,
+ cc: cc,
+ mc: mc,
+ dsc: dsc,
+ colc: colc,
+ pc: pc,
+ }, nil
+}
+
+type OrganizationHandler struct {
+ sc component.SpaceComponent
+ cc component.CodeComponent
+ mc component.ModelComponent
+ dsc component.DatasetComponent
+ colc component.CollectionComponent
+ pc component.PromptComponent
+}
+
+// GetOrganizationModels godoc
+// @Security ApiKey
+// @Summary Get organization models
+// @Description get organization models
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string true "current user name"
+// @Param per query int false "page size"
+// @Param page query int false "current page number"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Model,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/models [get]
+func (h *OrganizationHandler) Models(ctx *gin.Context) {
+ var req types.OrgModelsReq
+ req.Namespace = ctx.Param("namespace")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Page = page
+ req.PageSize = per
+ models, total, err := h.mc.OrgModels(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get org models", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get org models succeed", slog.String("org", req.Namespace))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": models,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetOrganizationDatasets godoc
+// @Security ApiKey
+// @Summary Get organization datasets
+// @Description get organization datasets
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string true "current user name"
+// @Param per query int false "page size"
+// @Param page query int false "current page number"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Dataset,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/datasets [get]
+func (h *OrganizationHandler) Datasets(ctx *gin.Context) {
+ var req types.OrgDatasetsReq
+ req.Namespace = ctx.Param("namespace")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Page = page
+ req.PageSize = per
+ datasets, total, err := h.dsc.OrgDatasets(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get org datasets", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get org datasets succeed", slog.String("org", req.Namespace))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": datasets,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetOrganizationCodes godoc
+// @Security ApiKey
+// @Summary Get organization codes
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string true "current user name"
+// @Param per query int false "page size"
+// @Param page query int false "current page number"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Code,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/codes [get]
+func (h *OrganizationHandler) Codes(ctx *gin.Context) {
+ var req types.OrgCodesReq
+ req.Namespace = ctx.Param("namespace")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Page = page
+ req.PageSize = per
+ datasets, total, err := h.cc.OrgCodes(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get org codes", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get org codes succeed", slog.String("org", req.Namespace))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": datasets,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetOrganizationSpaces godoc
+// @Security ApiKey
+// @Summary Get organization Spaces
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string true "current user name"
+// @Param per query int false "page size"
+// @Param page query int false "current page number"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Space,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/spaces [get]
+func (h *OrganizationHandler) Spaces(ctx *gin.Context) {
+ var req types.OrgSpacesReq
+ req.Namespace = ctx.Param("namespace")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Page = page
+ req.PageSize = per
+ datasets, total, err := h.sc.OrgSpaces(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get org spaces", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get org spaces succeed", slog.String("org", req.Namespace))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": datasets,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetOrganizationCollections godoc
+// @Security ApiKey
+// @Summary Get organization Collections
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string true "current user name"
+// @Param per query int false "page size"
+// @Param page query int false "current page number"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Collection,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/collections [get]
+func (h *OrganizationHandler) Collections(ctx *gin.Context) {
+ var req types.OrgCollectionsReq
+ req.Namespace = ctx.Param("namespace")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Page = page
+ req.PageSize = per
+ datasets, total, err := h.colc.OrgCollections(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get org collections", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "message": "OK",
+ "data": datasets,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetOrganizationPrompts godoc
+// @Security ApiKey
+// @Summary Get organization prompts
+// @Description get organization prompts
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string true "current user name"
+// @Param per query int false "page size"
+// @Param page query int false "current page number"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.PromptRes,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/prompts [get]
+func (h *OrganizationHandler) Prompts(ctx *gin.Context) {
+ var req types.OrgPromptsReq
+ req.Namespace = ctx.Param("namespace")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Page = page
+ req.PageSize = per
+ prompts, total, err := h.pc.OrgPrompts(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get org prompts", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "message": "OK",
+ "data": prompts,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+
+
package handler
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+type PromptHandler struct {
+ pc component.PromptComponent
+ sc component.SensitiveComponent
+ repo component.RepoComponent
+}
+
+func NewPromptHandler(cfg *config.Config) (*PromptHandler, error) {
+ promptComp, err := component.NewPromptComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create PromptComponent: %w", err)
+ }
+ sc, err := component.NewSensitiveComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create SensitiveComponent: %w", err)
+ }
+ repo, err := component.NewRepoComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create repo component: %w", err)
+ }
+ return &PromptHandler{
+ pc: promptComp,
+ sc: sc,
+ repo: repo,
+ }, nil
+}
+
+// GetVisiablePrompts godoc
+// @Security ApiKey
+// @Summary Get Visiable Prompt repos for current user
+// @Description get visiable Prompt repos for current user
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param search query string false "search text"
+// @Param task_tag query string false "filter by task tag"
+// @Param framework_tag query string false "filter by framework tag"
+// @Param license_tag query string false "filter by license tag"
+// @Param language_tag query string false "filter by language tag"
+// @Param sort query string false "sort by"
+// @Param source query string false "source" Enums(opencsg, huggingface, local)
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.PromptRes,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts [get]
+func (h *PromptHandler) Index(ctx *gin.Context) {
+ filter := new(types.RepoFilter)
+ filter.Tags = parseTagReqs(ctx)
+ filter.Username = httpbase.GetCurrentUser(ctx)
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format prompt list", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filter = getFilterFromContext(ctx, filter)
+ if !slices.Contains[[]string](Sorts, filter.Sort) {
+ msg := fmt.Sprintf("sort parameter must be one of %v", Sorts)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ if filter.Source != "" && !slices.Contains[[]string](Sources, filter.Source) {
+ msg := fmt.Sprintf("source parameter must be one of %v", Sources)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ prompts, total, err := h.pc.IndexPromptRepo(ctx, filter, per, page)
+ if err != nil {
+ slog.Error("Failed to get prompts dataset", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": prompts,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// ListPrompt godoc
+// @Security ApiKey
+// @Summary List prompts
+// @Description List prompts
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name} [get]
+func (h *PromptHandler) ListPrompt(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ detail, err := h.pc.Show(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get prompt detail", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ req := types.PromptReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ }
+ data, err := h.pc.ListPrompt(ctx, req)
+ if err != nil {
+ slog.Error("Failed to list prompts of repo", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "detail": detail,
+ "prompts": data,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+// GetPrompt godoc
+// @Security ApiKey
+// @Summary Get prompts by file
+// @Description Get prompts by file
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param file_path path string true "the file relative path"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/prompt/view/{file_path} [get]
+func (h *PromptHandler) GetPrompt(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filePath := ctx.Param("file_path")
+ if filePath == "" {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, "Bad request format")
+ return
+ }
+ req := types.PromptReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ Path: convertFilePathFromRoute(filePath),
+ }
+ data, err := h.pc.GetPrompt(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get prompt of repo", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// CreatePrompt godoc
+// @Security ApiKey
+// @Summary Create prompt in repo
+// @Description Create prompt in repo
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/prompt/file [post]
+func (h *PromptHandler) CreatePrompt(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var body *component.CreatePromptReq
+ if err := ctx.ShouldBindJSON(&body); err != nil {
+ slog.Error("Bad request prompt format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err = h.sc.CheckRequestV2(ctx, body)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req := types.PromptReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ }
+
+ data, err := h.pc.CreatePrompt(ctx, req, body)
+ if err != nil {
+ slog.Error("Failed to create prompt file of repo", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// UpdatePrompt godoc
+// @Security ApiKey
+// @Summary Update prompt in repo
+// @Description Update prompt in repo
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param file_path path string true "the file relative path"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/prompt/file/{file_path} [put]
+func (h *PromptHandler) UpdatePrompt(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filePath := ctx.Param("file_path")
+ if filePath == "" {
+ slog.Error("Bad request file path format", "error", err)
+ httpbase.BadRequest(ctx, "Bad request file path format")
+ return
+ }
+
+ var body *component.UpdatePromptReq
+ if err := ctx.ShouldBindJSON(&body); err != nil {
+ slog.Error("Bad request prompt format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err = h.sc.CheckRequestV2(ctx, body)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req := types.PromptReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ Path: convertFilePathFromRoute(filePath),
+ }
+ data, err := h.pc.UpdatePrompt(ctx, req, body)
+ if err != nil {
+ slog.Error("Failed to update prompt file of repo", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// DeletePrompt godoc
+// @Security ApiKey
+// @Summary Delete prompt in repo
+// @Description Delete prompt in repo
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param file_path path string true "the file relative path"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/prompt/file/{file_path} [delete]
+func (h *PromptHandler) DeletePrompt(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ filePath := ctx.Param("file_path")
+ if filePath == "" {
+ slog.Error("Bad request file path format", "error", err)
+ httpbase.BadRequest(ctx, "Bad request file path format")
+ return
+ }
+
+ req := types.PromptReq{
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ Path: convertFilePathFromRoute(filePath),
+ }
+ err = h.pc.DeletePrompt(ctx, req)
+ if err != nil {
+ slog.Error("Failed to remove prompt file of repo", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// NewConversation godoc
+// @Security ApiKey
+// @Summary Create new conversation
+// @Description Create new conversation
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param body body types.Conversation true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations [post]
+func (h *PromptHandler) NewConversation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var body *types.ConversationTitle
+ if err := ctx.ShouldBindJSON(&body); err != nil {
+ slog.Error("Bad request conversation body", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req := types.ConversationTitleReq{
+ CurrentUser: currentUser,
+ ConversationTitle: types.ConversationTitle{
+ Uuid: body.Uuid,
+ Title: body.Title,
+ },
+ }
+ resp, err := h.pc.NewConversation(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create conversation", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// ListConversation godoc
+// @Security ApiKey
+// @Summary List conversations of user
+// @Description List conversations of user
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations [get]
+func (h *PromptHandler) ListConversation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ data, err := h.pc.ListConversationsByUserID(ctx, currentUser)
+ if err != nil {
+ slog.Error("Failed to list conversations", slog.Any("currentUser", currentUser), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, data)
+}
+
+// GetConversation godoc
+// @Security ApiKey
+// @Summary Get a conversation by uuid
+// @Description Get a conversation by uuid
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param id path string true "conversation uuid"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations/{id} [get]
+func (h *PromptHandler) GetConversation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ uuid := ctx.Param("id")
+ if len(uuid) < 1 {
+ slog.Error("Bad request conversation uuid")
+ httpbase.BadRequest(ctx, "uuid is empty")
+ return
+ }
+ req := types.ConversationReq{
+ CurrentUser: currentUser,
+ Conversation: types.Conversation{
+ Uuid: uuid,
+ },
+ }
+ conversation, err := h.pc.GetConversation(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get conversation by id", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, conversation)
+}
+
+// SubmitMessage godoc
+// @Security ApiKey
+// @Summary Submit a conversation message
+// @Description Submit a conversation message
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param id path string true "conversation uuid"
+// @Param body body types.Conversation true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations/{id} [post]
+func (h *PromptHandler) SubmitMessage(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ uuid := ctx.Param("id")
+ if len(uuid) < 1 {
+ slog.Error("Bad request conversation uuid")
+ httpbase.BadRequest(ctx, "uuid is empty")
+ return
+ }
+ var body *types.Conversation
+ if err := ctx.ShouldBindJSON(&body); err != nil {
+ slog.Error("Bad request messsage body", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req := types.ConversationReq{
+ CurrentUser: currentUser,
+ Conversation: types.Conversation{
+ Uuid: uuid,
+ Message: body.Message,
+ Temperature: body.Temperature,
+ },
+ }
+
+ ch, err := h.pc.SubmitMessage(ctx, req)
+ if err != nil {
+ slog.Error("Failed to submit message", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ generatedText := ""
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Debug("generate respose end for context done", slog.Any("error", ctx.Request.Context().Err()))
+ res := types.Conversation{
+ Uuid: uuid,
+ Message: generatedText,
+ }
+ _, err = h.pc.SaveGeneratedText(ctx, res)
+ if err != nil {
+ slog.Error("fail to save generated message for request cancel", slog.Any("res", res), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ }
+ return
+ case data, ok := <-ch:
+ if ok {
+ if len(data) < 1 {
+ continue
+ }
+ data := strings.TrimSpace(strings.TrimPrefix(data, "data:"))
+ ctx.SSEvent("data", data)
+ ctx.Writer.Flush()
+ resp := types.LLMResponse{}
+ err := json.Unmarshal([]byte(data), &resp)
+ if err != nil {
+ slog.Warn("unmarshal llm response", slog.Any("data", data), slog.Any("error", err))
+ continue
+ }
+ if len(resp.Choices) < 1 {
+ continue
+ }
+ generatedText = fmt.Sprintf("%s%s", generatedText, resp.Choices[0].Delta.Content)
+ } else {
+ slog.Debug("stream channel closed")
+ res := types.Conversation{
+ Uuid: uuid,
+ Message: generatedText,
+ }
+ msg, err := h.pc.SaveGeneratedText(ctx, res)
+ if err != nil {
+ slog.Error("fail to save generated message for stream close", slog.Any("res", res), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ }
+ ctx.SSEvent("data", fmt.Sprintf("{\"msg_id\": %d}", msg.ID))
+ ctx.Writer.Flush()
+ return
+ }
+ }
+ }
+}
+
+// UpdateConversation godoc
+// @Security ApiKey
+// @Summary Update a conversation title
+// @Description Update a conversation title
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param id path string true "conversation uuid"
+// @Param body body types.ConversationTitle true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations/{id} [put]
+func (h *PromptHandler) UpdateConversation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ uuid := ctx.Param("id")
+ if len(uuid) < 1 {
+ slog.Error("Bad request conversation uuid")
+ httpbase.BadRequest(ctx, "uuid is empty")
+ return
+ }
+ var body *types.ConversationTitle
+ if err := ctx.ShouldBindJSON(&body); err != nil {
+ slog.Error("Bad request messsage body", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req := types.ConversationTitleReq{
+ CurrentUser: currentUser,
+ ConversationTitle: types.ConversationTitle{
+ Uuid: uuid,
+ Title: body.Title,
+ },
+ }
+ resp, err := h.pc.UpdateConversation(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update conversation", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// DeleteConversation godoc
+// @Security ApiKey
+// @Summary Delete a conversation
+// @Description Delete a conversation
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param id path string true "conversation uuid"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations/{id} [delete]
+func (h *PromptHandler) RemoveConversation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ uuid := ctx.Param("id")
+ if len(uuid) < 1 {
+ slog.Error("Bad request conversation uuid")
+ httpbase.BadRequest(ctx, "uuid is empty")
+ return
+ }
+ req := types.ConversationReq{
+ CurrentUser: currentUser,
+ Conversation: types.Conversation{
+ Uuid: uuid,
+ },
+ }
+ err := h.pc.RemoveConversation(ctx, req)
+ if err != nil {
+ slog.Error("Failed to remove conversation by id", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// LikeMessage godoc
+// @Security ApiKey
+// @Summary Like a conversation message
+// @Description Like a conversation message
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param uuid path string true "conversation uuid"
+// @Param id path string true "message id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations/{id}/message/{msgid}/like [put]
+func (h *PromptHandler) LikeMessage(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ uuid := ctx.Param("id")
+ if len(uuid) < 1 {
+ slog.Error("Bad request conversation uuid")
+ httpbase.BadRequest(ctx, "uuid is empty")
+ return
+ }
+ msgid := ctx.Param("msgid")
+ idInt, err := strconv.ParseInt(msgid, 10, 64)
+ if err != nil {
+ slog.Error("Bad request message id", slog.Any("msgid", msgid), slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req := types.ConversationMessageReq{
+ Uuid: uuid,
+ Id: idInt,
+ CurrentUser: currentUser,
+ }
+ err = h.pc.LikeConversationMessage(ctx, req)
+ if err != nil {
+ slog.Error("Failed to like conversation message", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// HateMessage godoc
+// @Security ApiKey
+// @Summary Hate a conversation message
+// @Description Hate a conversation message
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param uuid path string true "conversation uuid"
+// @Param id path string true "message id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/conversations/{id}/message/{msgid}/hate [put]
+func (h *PromptHandler) HateMessage(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ uuid := ctx.Param("id")
+ if len(uuid) < 1 {
+ slog.Error("Bad request conversation uuid")
+ httpbase.BadRequest(ctx, "uuid is empty")
+ return
+ }
+ msgid := ctx.Param("msgid")
+ idInt, err := strconv.ParseInt(msgid, 10, 64)
+ if err != nil {
+ slog.Error("Bad request message id", slog.Any("msgid", msgid), slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req := types.ConversationMessageReq{
+ Uuid: uuid,
+ Id: idInt,
+ CurrentUser: currentUser,
+ }
+ err = h.pc.HateConversationMessage(ctx, req)
+ if err != nil {
+ slog.Error("Failed to hate conversation message", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// PromptRelations godoc
+// @Security ApiKey
+// @Summary Get prompt related assets
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{data=types.Relations} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/relations [get]
+func (h *PromptHandler) Relations(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.pc.Relations(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get prompt relations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+// SetRelation godoc
+// @Security ApiKey
+// @Summary Set model relation for prompt
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param req body types.RelationModels true "set model relation"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/relations [put]
+func (h *PromptHandler) SetRelations(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RelationModels
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.CurrentUser = currentUser
+
+ err = h.pc.SetRelationModels(ctx, req)
+ if err != nil {
+ slog.Error("Failed to set models for prompt", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// AddModelRelation godoc
+// @Security ApiKey
+// @Summary add model relation for prompt
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param req body types.RelationModel true "add model relation"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/relations/model [post]
+func (h *PromptHandler) AddModelRelation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RelationModel
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.CurrentUser = currentUser
+
+ err = h.pc.AddRelationModel(ctx, req)
+ if err != nil {
+ slog.Error("Failed to add model for prompt", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteModelRelation godoc
+// @Security ApiKey
+// @Summary delete model relation for prompt
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param req body types.RelationModel true "delelet model relation"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/relations/model [delete]
+func (h *PromptHandler) DelModelRelation(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RelationModel
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+ req.CurrentUser = currentUser
+
+ err = h.pc.DelRelationModel(ctx, req)
+ if err != nil {
+ slog.Error("Failed to delete dataset for model", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// CreatePromptRepo godoc
+// @Security ApiKey
+// @Summary Create a new prompt repo
+// @Description create a new prompt repo
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user, the owner"
+// @Param body body types.CreatePromptRepoReq true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts [post]
+func (h *PromptHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.CreatePromptRepoReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request prompt repo format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ prompt, err := h.pc.CreatePromptRepo(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create prompt repo", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create prompt repo succeed", slog.String("prompt", prompt.Name))
+ respData := gin.H{
+ "data": prompt,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// UpdatePromptRepo godoc
+// @Security ApiKey
+// @Summary Update a exists prompt repo
+// @Description update a exists prompt repo
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the owner"
+// @Param body body types.UpdatePromptRepoReq true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name} [put]
+func (h *PromptHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdatePromptRepoReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+
+ prompt, err := h.pc.UpdatePromptRepo(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update prompt repo", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, prompt)
+}
+
+// DeletePromptRepo godoc
+// @Security ApiKey
+// @Summary Delete a exists prompt repo
+// @Description delete a exists prompt repo
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user, the owner"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name} [delete]
+func (h *PromptHandler) Delete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.pc.RemoveRepo(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete prompt repo", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetRepoBranches
+// @Security ApiKey
+// @Summary Get the branches of prompt repository
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Branch} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/branches [get]
+func (h *PromptHandler) Branches(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ req := &types.GetBranchesReq{
+ Namespace: namespace,
+ Name: name,
+ Per: per,
+ Page: page,
+ RepoType: types.PromptRepo,
+ CurrentUser: currentUser,
+ }
+ branches, err := h.repo.Branches(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get prompt repo branches", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, branches)
+}
+
+// GetRepoTags
+// @Security ApiKey
+// @Summary Get the tags of prompt repository
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=[]database.Tag} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/tags [get]
+func (h *PromptHandler) Tags(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ req := &types.GetTagsReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: types.PromptRepo,
+ CurrentUser: currentUser,
+ }
+ tags, err := h.repo.Tags(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get prompt repo tags", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, tags)
+}
+
+// UpdateRepoTags
+// @Security ApiKey
+// @Summary update the tags of a certain category
+// @Tags Prompt
+// @Accept json
+// @Produce json
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param current_user query string true "current user name"
+// @Param category path string true "tag category" Enums(task, license, framework, language, industry)
+// @Param tags body []string true "tag names in array"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /prompts/{namespace}/{name}/tags/{category} [post]
+func (h *PromptHandler) UpdateTags(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, httpbase.ErrorNeedLogin)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Failed update tags", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var tags []string
+ if err := ctx.ShouldBindJSON(&tags); err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("failed to unmarshal tags: %w", err).Error())
+ return
+ }
+ category := ctx.Param("category")
+
+ err = h.repo.UpdateTags(ctx, namespace, name, types.PromptRepo, category, currentUser, tags)
+ if err != nil {
+ slog.Error("Failed to update tags", slog.String("error", err.Error()), slog.String("category", category), slog.String("namespace", namespace), slog.String("name", name))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+func (h *PromptHandler) UpdateDownloads(ctx *gin.Context) {
+ var req *types.UpdateDownloadsReq
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Namespace = namespace
+ req.Name = name
+ req.RepoType = types.PromptRepo
+ date, err := time.Parse("2006-01-02", req.ReqDate)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Date = date
+
+ err = h.repo.UpdateDownloads(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update repo download count", slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name), slog.Time("date", date), slog.Int64("clone_count", req.CloneCount))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+// RecomHandler handles requests for repo recommendation
+type RecomHandler struct {
+ c component.RecomComponent
+}
+
+func NewRecomHandler(cfg *config.Config) (*RecomHandler, error) {
+ c, err := component.NewRecomComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create RecomComponent: %w", err)
+ }
+ return &RecomHandler{
+ c: c,
+ }, nil
+}
+
+// SetOpWeight godoc
+// @Security ApiKey
+// @Summary set op weight for repo recommendation
+// @Tags Recommendation
+// @Accept json
+// @Produce json
+// @Param body body handler.SetOpWeight.SetOpWeightReq true "json request body"
+// @Param current_user query string true "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /recom/opweight [post]
+func (h *RecomHandler) SetOpWeight(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ type SetOpWeightReq struct {
+ RepoID int64 `json:"repo_id" binding:"required"`
+ Weight int64 `json:"weight" binding:"required"`
+ }
+
+ var req SetOpWeightReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("unmarshal body failed, %w", err).Error())
+ return
+ }
+
+ err := h.c.SetOpWeight(ctx, req.RepoID, req.Weight, currentUser)
+ if err != nil {
+ slog.Error("failed to set op weight", slog.Int64("repo_id", req.RepoID), slog.Int64("weight", req.Weight),
+ slog.Any("error", err))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to set op weight for repository %d, err:%w", req.RepoID, err))
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "bytes"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewRepoHandler(config *config.Config) (*RepoHandler, error) {
+ uc, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &RepoHandler{
+ c: uc,
+ }, nil
+}
+
+type RepoHandler struct {
+ c component.RepoComponent
+}
+
+// CreateRepoFile godoc
+// @Security ApiKey
+// @Summary Create a new file in repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param file_path path string true "the new file relative path"
+// @Param current_user query string false "current user name"
+// @Param req body types.CreateFileReq true "create file request"
+// @Success 200 {object} types.ResponseWithTotal{data=types.CreateFileResp} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/raw/{file_path} [post]
+func (h *RepoHandler) CreateFile(ctx *gin.Context) {
+ userName := httpbase.GetCurrentUser(ctx)
+ if userName == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.CreateFileReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Username = userName
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ req.Namespace = namespace
+ req.Name = name
+ req.FilePath = filePath
+ req.RepoType = common.RepoTypeFromContext(ctx)
+ req.CurrentUser = userName
+
+ resp, err := h.c.CreateFile(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create repo file", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create repo file succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name))
+ httpbase.OK(ctx, resp)
+}
+
+// UpdateRepoFile godoc
+// @Security ApiKey
+// @Summary Update existing file in repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param file_path path string true "the new file relative path"
+// @Param current_user query string false "current user name"
+// @Param req body types.UpdateFileReq true "create file request"
+// @Success 200 {object} types.ResponseWithTotal{data=types.UpdateFileResp} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/raw/{file_path} [put]
+func (h *RepoHandler) UpdateFile(ctx *gin.Context) {
+ userName := httpbase.GetCurrentUser(ctx)
+ if userName == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdateFileReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Username = userName
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ req.Namespace = namespace
+ req.Name = name
+ req.FilePath = filePath
+ req.RepoType = common.RepoTypeFromContext(ctx)
+ req.CurrentUser = userName
+
+ resp, err := h.c.UpdateFile(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update repo file", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Update repo file succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name))
+ httpbase.OK(ctx, resp)
+}
+
+// GetRepoCommits godoc
+// @Security ApiKey
+// @Summary Get all commits of repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Commit} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/commits [get]
+func (h *RepoHandler) Commits(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ ref := ctx.Query("ref")
+ req := &types.GetCommitsReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: ref,
+ Per: per,
+ Page: page,
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ commits, pageOpt, err := h.c.Commits(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get repo commits", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo commits succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("ref", req.Ref))
+ resData := gin.H{
+ "commits": commits,
+ "total": pageOpt.Total,
+ "page_count": pageOpt.PageCount,
+ }
+ httpbase.OK(ctx, resData)
+}
+
+// GetRepoLastCommit godoc
+// @Security ApiKey
+// @Summary Get the last commit of repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=types.Commit} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/last_commit [get]
+func (h *RepoHandler) LastCommit(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ ref := ctx.Query("ref")
+ req := &types.GetCommitsReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: ref,
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ commit, err := h.c.LastCommit(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrForbidden) {
+ httpbase.ForbiddenError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get repo last commit", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo last commit succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("ref", req.Ref))
+ httpbase.OK(ctx, commit)
+}
+
+// GetRepoFileContent godoc
+// @Security ApiKey
+// @Summary Get the last commit of repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param file_path path string true "file path"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=string} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/raw/{file_path} [get]
+func (h *RepoHandler) FileRaw(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ req := &types.GetFileReq{
+ Namespace: namespace,
+ Name: name,
+ Path: filePath,
+ Ref: ctx.Query("ref"),
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ raw, err := h.c.FileRaw(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get repo file raw", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo file raw succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("path", req.Path), slog.String("ref", req.Ref))
+ httpbase.OK(ctx, raw)
+}
+
+// GetRepoFileContent godoc
+// @Security ApiKey or JWT
+// @Summary Get the repo file information like size, content, sha etc
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param file_path path string true "file path"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=types.File} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/blob/{file_path} [get]
+func (h *RepoHandler) FileInfo(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ req := &types.GetFileReq{
+ Namespace: namespace,
+ Name: name,
+ Path: filePath,
+ Ref: ctx.Query("ref"),
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ file, err := h.c.FileInfo(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get repo file info", slog.Any("req", req), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo file info succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("path", req.Path), slog.String("ref", req.Ref))
+ httpbase.OK(ctx, file)
+}
+
+// DownloadRepoFile godoc
+// @Security ApiKey or JWT
+// @Summary Download a repo file [Depricated: use 'resolve' api instead]
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param file_path path string true "file path"
+// @Param ref query string false "branch or tag"
+// @Param save_as query string false "file name to save as"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=object} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/download/{file_path} [get]
+func (h *RepoHandler) DownloadFile(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ req := &types.GetFileReq{
+ Namespace: namespace,
+ Name: name,
+ Path: filePath,
+ Ref: ctx.Query("ref"),
+ Lfs: false,
+ SaveAs: ctx.Query("save_as"),
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ if ctx.Query("lfs") != "" {
+ req.Lfs, err = strconv.ParseBool(ctx.Query("lfs"))
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ }
+ reader, size, url, err := h.c.DownloadFile(ctx, req, currentUser)
+ if err != nil {
+ slog.Error("Failed to download repo file", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ if req.Lfs {
+ httpbase.OK(ctx, url)
+ } else {
+ slog.Info("Download repo file succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("path", req.Path), slog.String("ref", req.Ref))
+ fileName := path.Base(req.Path)
+ ctx.Header("Content-Type", "application/octet-stream")
+ ctx.Header("Content-Disposition", `attachment; filename="`+fileName+`"`)
+ ctx.Header("Content-Length", strconv.FormatInt(size, 10))
+ _, err = io.Copy(ctx.Writer, reader)
+ if err != nil {
+ slog.Error("Failed to download repo file", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ }
+}
+
+// GetRepoBranches
+// @Security ApiKey
+// @Summary Get the branches of repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Branch} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/branches [get]
+func (h *RepoHandler) Branches(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ req := &types.GetBranchesReq{
+ Namespace: namespace,
+ Name: name,
+ Per: per,
+ Page: page,
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ branches, err := h.c.Branches(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get repo branches", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo branches succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name))
+ httpbase.OK(ctx, branches)
+}
+
+// GetRepoTags
+// @Security ApiKey
+// @Summary Get the tags of repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=[]database.Tag} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/tags [get]
+func (h *RepoHandler) Tags(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ req := &types.GetTagsReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ tags, err := h.c.Tags(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get repo tags", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo tags succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name))
+ httpbase.OK(ctx, tags)
+}
+
+// UpdateRepoTags
+// @Security ApiKey
+// @Summary update the tags of a certain category
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param current_user query string true "current user name"
+// @Param category path string true "tag category" Enums(task, license, framework, language, industry)
+// @Param tags body []string true "tag names in array"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/tags/{category} [post]
+func (h *RepoHandler) UpdateTags(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, httpbase.ErrorNeedLogin)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Failed update tags", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var tags []string
+ if err := ctx.ShouldBindJSON(&tags); err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("failed to unmarshal tags: %w", err).Error())
+ return
+ }
+ category := ctx.Param("category")
+ repoType := common.RepoTypeFromContext(ctx)
+ err = h.c.UpdateTags(ctx, namespace, name, repoType, category, currentUser, tags)
+ if err != nil {
+ slog.Error("Failed to update tags", slog.String("error", err.Error()), slog.String("category", category), slog.String("repo_type", string(repoType)),
+ slog.String("namespace", namespace), slog.String("name", name))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// GetRepoTree godoc
+// @Security ApiKey
+// @Summary Get repository file tree
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param path query string false "root dir"
+// @Param ref query string false "branch or tag"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.File} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/tree [get]
+func (h *RepoHandler) Tree(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ req := &types.GetFileReq{
+ Namespace: namespace,
+ Name: name,
+ Path: ctx.Query("path"),
+ Ref: ctx.Query("ref"),
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ tree, err := h.c.Tree(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrForbidden) {
+ httpbase.ForbiddenError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get repo file tree", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get repo file tree succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("path", req.Path), slog.String("ref", req.Ref))
+ httpbase.OK(ctx, tree)
+}
+
+func (h *RepoHandler) UpdateDownloads(ctx *gin.Context) {
+ var req *types.UpdateDownloadsReq
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Namespace = namespace
+ req.Name = name
+ req.RepoType = common.RepoTypeFromContext(ctx)
+ date, err := time.Parse("2006-01-02", req.ReqDate)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Date = date
+
+ err = h.c.UpdateDownloads(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update repo download count", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name), slog.Time("date", date), slog.Int64("clone_count", req.CloneCount))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Update repo download count succeed", slog.String("repo_type", string(req.RepoType)), slog.String("namespace", namespace), slog.String("name", name), slog.Int64("clone_count", req.CloneCount))
+ httpbase.OK(ctx, nil)
+}
+
+// IncrDownloads godoc
+// @Security ApiKey
+// @Summary Increase repo download count by 1
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Success 200 {object} types.Response "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/incr_downloads [put]
+func (h *RepoHandler) IncrDownloads(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ repoType := common.RepoTypeFromContext(ctx)
+
+ err = h.c.IncrDownloads(ctx, repoType, namespace, name)
+ if err != nil {
+ slog.Error("Failed to increase repo download count", slog.String("repo_type", string(repoType)), slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("increase repo download count succeed", slog.String("repo_type", string(repoType)), slog.String("namespace", namespace), slog.String("name", name))
+ httpbase.OK(ctx, nil)
+}
+
+func (h *RepoHandler) UploadFile(ctx *gin.Context) {
+ userName := httpbase.GetCurrentUser(ctx)
+ if userName == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ form, _ := ctx.MultipartForm()
+ fileList := form.File["file"]
+ paths := form.Value["file_path"]
+ var message, branch string
+ if len(form.Value["message"]) > 0 {
+ message = form.Value["message"][0]
+ }
+ if len(form.Value["branch"]) > 0 {
+ branch = form.Value["branch"][0]
+ }
+ var buf bytes.Buffer
+ for idx, file := range fileList {
+ openedFile, err := file.Open()
+ if err != nil {
+ slog.Error("Error opening uploaded file", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ buf.Reset()
+ _, err = io.Copy(&buf, openedFile)
+ if err != nil {
+ slog.Info("Error encodeing uploaded file", "error", err, slog.String("file_name", file.Filename))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ originalBytes := buf.Bytes()
+ base64Content := base64.StdEncoding.EncodeToString(originalBytes)
+ openedFile.Close()
+ filePath := paths[idx]
+
+ upload := &types.CreateFileReq{
+ Username: userName,
+ Namespace: namespace,
+ Name: name,
+ FilePath: filePath,
+ Content: base64Content,
+ RepoType: common.RepoTypeFromContext(ctx),
+ Message: message,
+ Branch: branch,
+ OriginalContent: originalBytes,
+ CurrentUser: userName,
+ }
+ err = h.c.UploadFile(ctx, upload)
+ if err != nil {
+ slog.Error("Failed to upload repo file", slog.String("repo_type", string(upload.RepoType)), slog.Any("error", err), slog.String("file_path", filePath))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Upload file succeed", slog.String("repo_type", string(upload.RepoType)), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)),
+ slog.String("file_name", file.Filename))
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+func (h *RepoHandler) SDKListFiles(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ref := ctx.Param("ref")
+ mappedBranch := ctx.Param("branch_mapped")
+ if mappedBranch != "" {
+ ref = mappedBranch
+ }
+ files, err := h.c.SDKListFiles(ctx, common.RepoTypeFromContext(ctx), namespace, name, ref, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ slog.Error("permission denied when accessing repo", slog.String("repo_type", string(common.RepoTypeFromContext(ctx))), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ if errors.Is(err, component.ErrNotFound) {
+ slog.Error("repo not found", slog.String("repo_type", string(common.RepoTypeFromContext(ctx))), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.NotFoundError(ctx, err)
+ return
+ }
+ slog.Error("Error listing repo files", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ ctx.JSON(http.StatusOK, files)
+}
+
+func (h *RepoHandler) SDKDownload(ctx *gin.Context) {
+ h.handleDownload(ctx, false)
+}
+
+// DownloadRepoFile godoc
+// @Security ApiKey
+// @Summary Download a rep file
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,dataset,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param file_path path string true "file path"
+// @Param ref query string true "branch or tag"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.ResponseWithTotal{data=string} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/resolve/{file_path} [get]
+func (h *RepoHandler) ResolveDownload(ctx *gin.Context) {
+ h.handleDownload(ctx, true)
+}
+
+func (h *RepoHandler) HeadSDKDownload(ctx *gin.Context) {
+ var repoCommit string
+ currentUser := httpbase.GetCurrentUser(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ branch := ctx.Param("branch")
+ mappedBranch := ctx.Param("branch_mapped")
+ if mappedBranch != "" {
+ branch = mappedBranch
+ }
+ req := &types.GetFileReq{
+ Namespace: namespace,
+ Name: name,
+ Path: filePath,
+ Ref: branch,
+ Lfs: false,
+ SaveAs: filepath.Base(filePath),
+ RepoType: common.RepoTypeFromContext(ctx),
+ }
+
+ file, commit, err := h.c.HeadDownloadFile(ctx, req, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ slog.Error("permission denied when accessing repo head", slog.String("repo_type", string(req.RepoType)), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+
+ if errors.Is(err, component.ErrNotFound) {
+ slog.Error("repo not found head", slog.String("repo_type", string(common.RepoTypeFromContext(ctx))), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.NotFoundError(ctx, err)
+ return
+ }
+
+ slog.Error("Failed to download repo file head", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ if commit != nil {
+ repoCommit = commit.ID
+ }
+
+ slog.Info("Head download repo file succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("path", req.Path), slog.String("ref", req.Ref),
+ slog.Int64("contentLength", file.Size))
+ ctx.Header("Content-Length", strconv.Itoa(int(file.Size)))
+ ctx.Header("X-Repo-Commit", repoCommit)
+ ctx.Header("ETag", file.SHA)
+ ctx.Status(http.StatusOK)
+}
+
+func (h *RepoHandler) handleDownload(ctx *gin.Context, isResolve bool) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ var (
+ namespace string
+ name string
+ branch string
+ reader io.ReadCloser
+ size int64
+ url string
+ contentLength int64
+ err error
+ )
+ namespace, name, err = common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ filePath := ctx.Param("file_path")
+ filePath = convertFilePathFromRoute(filePath)
+ if isResolve {
+ branch = ctx.Query("ref")
+ } else {
+ branch = ctx.Param("branch")
+ }
+ mappedBranch := ctx.Param("branch_mapped")
+ if mappedBranch != "" {
+ branch = mappedBranch
+ }
+ req := &types.GetFileReq{
+ Namespace: namespace,
+ Name: name,
+ Path: filePath,
+ Ref: branch,
+ Lfs: false,
+ SaveAs: filepath.Base(filePath),
+ RepoType: common.RepoTypeFromContext(ctx),
+ }
+ // TODO:move the check into SDKDownloadFile, and can return the file content as we get all the content before check lfs pointer
+ lfs, contentLength, err := h.c.IsLfs(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrNotFound) {
+ slog.Error("repo not found", slog.String("repo_type", string(common.RepoTypeFromContext(ctx))), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.NotFoundError(ctx, err)
+ return
+ }
+
+ slog.Error("Filed to lfs information", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ req.Lfs = lfs
+
+ if contentLength > 0 {
+ // file content is not empty, download it directly
+ reader, size, url, err = h.c.SDKDownloadFile(ctx, req, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ slog.Error("permission denied when accessing repo", slog.String("repo_type", string(req.RepoType)), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+
+ if errors.Is(err, component.ErrNotFound) {
+ slog.Error("repo not found", slog.String("repo_type", string(common.RepoTypeFromContext(ctx))), slog.Any("path", fmt.Sprintf("%s/%s", namespace, name)))
+ httpbase.NotFoundError(ctx, err)
+ return
+ }
+
+ slog.Error("Failed to download repo file", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ }
+
+ if req.Lfs {
+ ctx.Redirect(http.StatusMovedPermanently, url)
+ } else {
+ slog.Info("Download repo file succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("path", req.Path), slog.String("ref", req.Ref), slog.Any("Content-Length", size))
+ fileName := path.Base(req.Path)
+ ctx.Header("Content-Type", "application/octet-stream")
+ ctx.Header("Content-Disposition", `attachment; filename="`+fileName+`"`)
+ ctx.Header("Content-Length", strconv.FormatInt(size, 10))
+ if contentLength > 0 {
+ _, err = io.Copy(ctx.Writer, reader)
+ }
+ if err != nil {
+ slog.Error("Failed to download repo file", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ }
+}
+
+// GetRepoCommitDiff godoc
+// @Security ApiKey
+// @Summary Get commit diff of repository and data field of response need to be decode with base64
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param commit_id path string true "commit id"
+// @Param current_user query string false "current user name"
+// @Success 200 {object} types.Response{data=types.CommitResponse} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/commit/{commit_id} [get]
+func (h *RepoHandler) CommitWithDiff(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ commitID := ctx.Param("commit_id")
+ req := &types.GetCommitsReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: commitID,
+ RepoType: common.RepoTypeFromContext(ctx),
+ CurrentUser: currentUser,
+ }
+ commit, err := h.c.GetCommitWithDiff(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get repo with commit diff", slog.String("repo_type", string(req.RepoType)), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get repo commit with diff succeed", slog.String("repo_type", string(req.RepoType)), slog.String("name", name), slog.String("commit id", req.Ref))
+ // client need base64 decode for diff, for example: echo <diff> | base64 -d
+ httpbase.OK(ctx, commit)
+}
+
+// CreateMirror godoc
+// @Security ApiKey
+// @Summary Create mirror for a existing repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param body body types.CreateMirrorParams true "body"
+// @Success 200 {object} types.Response{data=database.Mirror} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror [post]
+func (h *RepoHandler) CreateMirror(ctx *gin.Context) {
+ var mirrorReq types.CreateMirrorReq
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ if err = ctx.ShouldBindJSON(&mirrorReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ sourceRepoPath, err := getSourceRepoPathFromSourceUrl(mirrorReq.SourceUrl)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ mirrorReq.Namespace = namespace
+ mirrorReq.Name = name
+ mirrorReq.RepoType = common.RepoTypeFromContext(ctx)
+ mirrorReq.CurrentUser = currentUser
+ mirrorReq.SourceRepoPath = sourceRepoPath
+ mirror, err := h.c.CreateMirror(ctx, mirrorReq)
+ if err != nil {
+ slog.Error("Failed to create mirror for", slog.String("repo_type", string(mirrorReq.RepoType)), slog.String("path", fmt.Sprintf("%s/%s", mirrorReq.Namespace, mirrorReq.Name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, mirror)
+}
+
+// MirrorFromSaas godoc
+// @Security ApiKey
+// @Summary Mirror repo from OpenCSG Saas(only on-premises)
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Success 200 {object} types.Response{data=database.Mirror} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror_from_saas [post]
+func (h *RepoHandler) MirrorFromSaas(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ repoType := common.RepoTypeFromContext(ctx)
+ if !strings.HasPrefix(namespace, types.OpenCSGPrefix) {
+ httpbase.BadRequest(ctx, "Repo could not be mirrored")
+ return
+ }
+ err = h.c.MirrorFromSaas(ctx, namespace, name, currentUser, repoType)
+ if err != nil {
+ slog.Error("Failed to create mirror for", slog.String("repo_type", string(repoType)), slog.String("path", fmt.Sprintf("%s/%s", namespace, name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetMirror godoc
+// @Security ApiKey
+// @Summary Get a mirror
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{data=database.Mirror} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror [get]
+func (h *RepoHandler) GetMirror(ctx *gin.Context) {
+ var mirrorReq types.GetMirrorReq
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ mirrorReq.Namespace = namespace
+ mirrorReq.Name = name
+ mirrorReq.RepoType = common.RepoTypeFromContext(ctx)
+ mirrorReq.CurrentUser = currentUser
+ mirror, err := h.c.GetMirror(ctx, mirrorReq)
+ if err != nil {
+ slog.Error("Failed to get mirror of", slog.String("repo_type", string(mirrorReq.RepoType)), slog.String("path", fmt.Sprintf("%s/%s", mirrorReq.Namespace, mirrorReq.Name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, mirror)
+}
+
+// UpdateMirror godoc
+// @Security ApiKey
+// @Summary Update a mirror for a existing repository
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Param body body types.UpdateMirrorParams true "body"
+// @Success 200 {object} types.Response{data=database.Mirror} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror [put]
+func (h *RepoHandler) UpdateMirror(ctx *gin.Context) {
+ var mirrorReq types.UpdateMirrorReq
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ if err = ctx.ShouldBindJSON(&mirrorReq); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ sourceRepoPath, err := getSourceRepoPathFromSourceUrl(mirrorReq.SourceUrl)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ mirrorReq.SourceRepoPath = sourceRepoPath
+ mirrorReq.Namespace = namespace
+ mirrorReq.Name = name
+ mirrorReq.RepoType = common.RepoTypeFromContext(ctx)
+ mirrorReq.CurrentUser = currentUser
+ mirror, err := h.c.UpdateMirror(ctx, mirrorReq)
+ if err != nil {
+ slog.Error("Failed to update mirror for", slog.String("repo_type", string(mirrorReq.RepoType)), slog.String("path", fmt.Sprintf("%s/%s", mirrorReq.Namespace, mirrorReq.Name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, mirror)
+}
+
+// DeleteMirror godoc
+// @Security ApiKey
+// @Summary Delete a mirror
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror [delete]
+func (h *RepoHandler) DeleteMirror(ctx *gin.Context) {
+ var mirrorReq types.DeleteMirrorReq
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ mirrorReq.Namespace = namespace
+ mirrorReq.Name = name
+ mirrorReq.RepoType = common.RepoTypeFromContext(ctx)
+ mirrorReq.CurrentUser = currentUser
+ err = h.c.DeleteMirror(ctx, mirrorReq)
+ if err != nil {
+ slog.Error("Failed to delete mirror of", slog.String("repo_type", string(mirrorReq.RepoType)), slog.String("path", fmt.Sprintf("%s/%s", mirrorReq.Namespace, mirrorReq.Name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// RuntimeFramework godoc
+// @Security ApiKey
+// @Summary List repo runtime framework
+// @Description List repo runtime framework
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2, 4) default(1)
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/runtime_framework [get]
+func (h *RepoHandler) RuntimeFrameworkList(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ slog.Debug("list runtime framework", slog.Any("namespace", namespace), slog.Any("name", name))
+ repoType := common.RepoTypeFromContext(ctx)
+ if repoType == types.UnknownRepo {
+ slog.Error("Bad request of repo type")
+ httpbase.BadRequest(ctx, "Bad request of repo type")
+ return
+ }
+ response, err := h.c.ListRuntimeFramework(ctx, repoType, namespace, name, deployType)
+ if err != nil {
+ slog.Error("fail to list runtime framework", slog.String("error", err.Error()))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, response)
+}
+
+// CreateRuntimeFramework godoc
+// @Security ApiKey
+// @Summary Create runtime framework
+// @Description create runtime framework
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param body body types.RuntimeFrameworkReq true "body"
+// @Success 200 {object} types.RuntimeFramework "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/runtime_framework [post]
+func (h *RepoHandler) RuntimeFrameworkCreate(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ slog.Debug("create runtime framework", slog.Any("namespace", namespace), slog.Any("name", name))
+ var req types.RuntimeFrameworkReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ frame, err := h.c.CreateRuntimeFramework(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create runtime framework", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, frame)
+}
+
+// UpdateRuntimeFramework godoc
+// @Security ApiKey
+// @Summary Update runtime framework
+// @Description Update runtime framework
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Param body body types.RuntimeFrameworkReq true "body"
+// @Success 200 {object} types.RuntimeFramework "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/runtime_framework/{id} [put]
+func (h *RepoHandler) RuntimeFrameworkUpdate(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ slog.Debug("update runtime framework", slog.Any("namespace", namespace), slog.Any("name", name))
+ var req types.RuntimeFrameworkReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request url format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ frame, err := h.c.UpdateRuntimeFramework(ctx, id, &req)
+ if err != nil {
+ slog.Error("Failed to update runtime framework", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, frame)
+}
+
+// DeleteRuntimeFramework godoc
+// @Security ApiKey
+// @Summary Delete a exist RuntimeFramework
+// @Description delete a exist RuntimeFramework
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path int true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/runtime_framework/{id} [delete]
+func (h *RepoHandler) RuntimeFrameworkDelete(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = h.c.DeleteRuntimeFramework(ctx, id)
+ if err != nil {
+ slog.Error("Failed to delete runtime framework", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeployList godoc
+// @Security ApiKey
+// @Summary List repo deploys
+// @Description List repo deploys
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/run [get]
+func (h *RepoHandler) DeployList(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ repoType := common.RepoTypeFromContext(ctx)
+ response, err := h.c.ListDeploy(ctx, repoType, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("fail to list deploy", slog.String("error", err.Error()), slog.Any("repotype", repoType), slog.Any("namespace", namespace), slog.Any("name", name))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, response)
+}
+
+// DeployDetail godoc
+// @Security ApiKey
+// @Summary Get repo deploy detail
+// @Description Get repo deploy detail
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 401 {object} types.APIUnauthorized "Permission denied"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/run/{id} [get]
+func (h *RepoHandler) DeployDetail(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ repoType := common.RepoTypeFromContext(ctx)
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ detailReq := types.DeployActReq{
+ RepoType: repoType,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ }
+ if repoType == types.SpaceRepo {
+ detailReq.DeployType = types.SpaceType
+ } else if repoType == types.ModelRepo {
+ detailReq.DeployType = types.InferenceType
+ }
+
+ response, err := h.c.DeployDetail(ctx, detailReq)
+ if err != nil {
+ slog.Error("fail to deploy detail", slog.String("error", err.Error()), slog.Any("repotype", repoType), slog.Any("namespace", namespace), slog.Any("name", name), slog.Any("deploy id", deployID))
+ var pErr *types.PermissionError
+ if errors.As(err, &pErr) {
+ httpbase.UnauthorizedError(ctx, err)
+ } else {
+ httpbase.ServerError(ctx, err)
+ }
+ return
+ }
+
+ httpbase.OK(ctx, response)
+}
+
+// GetInferenceLogs godoc
+// @Security ApiKey
+// @Summary get deploy instance logs
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "id"
+// @Param instance path string true "instance"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 401 {object} types.APIUnauthorized "Permission denied"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/run/{id}/logs/{instance} [get]
+func (h *RepoHandler) DeployInstanceLogs(ctx *gin.Context) {
+ if ctx.Query("test") == "true" {
+ h.testLogs(ctx)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace and name from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ repoType := common.RepoTypeFromContext(ctx)
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ instance := ctx.Param("instance")
+ if len(instance) < 1 {
+ httpbase.UnauthorizedError(ctx, errors.New("fail to get deploy instance"))
+ return
+ }
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ logReq := types.DeployActReq{
+ RepoType: repoType,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.InferenceType,
+ InstanceName: instance,
+ }
+
+ // user http request context instead of gin context, so that server knows the life cycle of the request
+ logReader, err := h.c.DeployInstanceLogs(ctx.Request.Context(), logReq)
+ if err != nil {
+ var pErr *types.PermissionError
+ if errors.As(err, &pErr) {
+ httpbase.UnauthorizedError(ctx, err)
+ } else {
+ slog.Error("Failed to get deploy instance logs", slog.Any("logReq", logReq), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ }
+ return
+ }
+
+ if logReader.RunLog() == nil {
+ httpbase.ServerError(ctx, errors.New("don't find any deploy instance log"))
+ return
+ }
+
+ // to quickly respond the http request
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("repo handler logs request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ case data, ok := <-logReader.RunLog():
+ if ok {
+ ctx.SSEvent("Container", string(data))
+ ctx.Writer.Flush()
+ }
+ }
+ }
+}
+
+func (h *RepoHandler) testLogs(ctx *gin.Context) {
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ return
+ default:
+ ctx.SSEvent("Container", "test run log message")
+ ctx.Writer.Flush()
+ }
+ time.Sleep(time.Second * 5)
+ }
+}
+
+func getSourceRepoPathFromSourceUrl(sourceUrl string) (string, error) {
+ parsedURL, err := url.Parse(sourceUrl)
+ if err != nil {
+ return "", err
+ }
+
+ // Remove leading and trailing slashes
+ path := strings.Trim(parsedURL.Path, "/")
+
+ // Remove ".git" suffix
+ path = strings.TrimSuffix(path, ".git")
+
+ return path, nil
+}
+
+// GetDeployStatus godoc
+// @Security JWT token
+// @Summary get deploy status
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,spaces" Enums(models,spaces)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "deploy id"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 401 {object} types.APIUnauthorized "Permission denied"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/run/{id}/status [get]
+func (h *RepoHandler) DeployStatus(ctx *gin.Context) {
+ if ctx.Query("test") == "true" {
+ h.testStatus(ctx)
+ return
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ repoType := common.RepoTypeFromContext(ctx)
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ statusReq := types.DeployActReq{
+ RepoType: repoType,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.InferenceType,
+ }
+
+ allow, err := h.c.AllowAccessDeploy(ctx, statusReq)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ var pErr *types.PermissionError
+ if errors.As(err, &pErr) {
+ httpbase.UnauthorizedError(ctx, err)
+ } else {
+ httpbase.ServerError(ctx, err)
+ }
+ return
+ }
+
+ if !allow {
+ slog.Info("user not allowed to query deploy status", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser), slog.Any("deploy_id", deployID))
+ }
+
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("deploy handler status request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ default:
+ time.Sleep(time.Second * 5)
+ // user http request context instead of gin context, so that server knows the life cycle of the request
+ _, status, instances, err := h.c.DeployStatus(ctx.Request.Context(), repoType, namespace, name, deployID)
+ if err != nil {
+ slog.Error("failed to get deploy status", slog.Any("error", err), slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("deploy_id", deployID))
+ ctx.SSEvent("error", err.Error())
+ } else {
+ eventData := &types.ModelStatusEventData{
+ Status: status,
+ Details: instances,
+ }
+ ctx.SSEvent("status", eventData)
+
+ }
+ ctx.Writer.Flush()
+ }
+ }
+}
+
+// SyncMirror godoc
+// @Security ApiKey
+// @Summary Triggers the mirror synchronization
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror/sync [post]
+func (h *RepoHandler) SyncMirror(ctx *gin.Context) {
+ repoType := common.RepoTypeFromContext(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ err = h.c.SyncMirror(ctx, repoType, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to sync mirror for", slog.String("repo_type", string(repoType)), slog.String("path", fmt.Sprintf("%s/%s", namespace, name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetMirrorProgress godoc
+// @Security ApiKey
+// @Summary Get Mirror sync progress
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models,datasets,codes or spaces" Enums(models,datasets,codes,spaces)
+// @Param namespace path string true "repo owner name"
+// @Param name path string true "repo name"
+// @Success 200 {object} types.Response{data=types.LFSSyncProgressResp} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/mirror/progress [get]
+func (h *RepoHandler) MirrorProgress(ctx *gin.Context) {
+ repoType := common.RepoTypeFromContext(ctx)
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ progress, err := h.c.MirrorProgress(ctx, repoType, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to get mirror progress for", slog.String("repo_type", string(repoType)), slog.String("path", fmt.Sprintf("%s/%s", namespace, name)), "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, progress)
+}
+
+func (h *RepoHandler) testStatus(ctx *gin.Context) {
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("deploy handler status request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ default:
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Building")
+ ctx.Writer.Flush()
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Running")
+ ctx.Writer.Flush()
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Sleeping")
+ ctx.Writer.Flush()
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Stopped")
+ ctx.Writer.Flush()
+ }
+ }
+}
+
+// DeployUpdate godoc
+// @Security ApiKey
+// @Summary Update deploy parameters
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models" Enums(models)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "deploy id"
+// @Param current_user query string true "current_user"
+// @Param body body types.DeployUpdateReq true "deploy setting of inference"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /{repo_type}/{namespace}/{name}/run/{id} [put]
+func (h *RepoHandler) DeployUpdate(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace and name from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ allow, err := h.c.AllowAdminAccess(ctx, types.ModelRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err, slog.Any("currentUser", currentUser), slog.Any("namespace", name), slog.Any("name", name))
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+ if !allow {
+ slog.Info("user not allowed to update deploy", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ httpbase.UnauthorizedError(ctx, errors.New("user not allowed to update deploy"))
+ return
+ }
+
+ var req *types.DeployUpdateReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err, slog.Any("request.body", ctx.Request.Body))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if req.MinReplica != nil && req.MaxReplica != nil {
+ err = Validate.Struct(req)
+ if err != nil {
+ slog.Error("Bad request setting for deploy", slog.Any("req", *req), slog.Any("err", err))
+ httpbase.BadRequest(ctx, fmt.Sprintf("Bad request setting for deploy, %v", err))
+ return
+ }
+ }
+
+ repoType := common.RepoTypeFromContext(ctx)
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", slog.Any("error", err), slog.Any("id", ctx.Param("id")))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ updateReq := types.DeployActReq{
+ RepoType: repoType,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.InferenceType,
+ }
+ err = h.c.DeployUpdate(ctx, updateReq, req)
+ if err != nil {
+ slog.Error("failed to update deploy", slog.String("namespace", namespace), slog.String("name", name), slog.Any("username", currentUser), slog.Int64("deploy_id", deployID), slog.Any("error", err))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to update deploy, %w", err))
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// RuntimeFrameworkListWithType godoc
+// @Security ApiKey
+// @Summary List repo runtime framework
+// @Description List repo runtime framework
+// @Tags Repository
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models" Enums(models)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string false "current user"
+// @Param deploy_type query int false "deploy_type" Enums(0, 1, 2) default(1)
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/runtime_framework [get]
+func (h *RepoHandler) RuntimeFrameworkListWithType(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ repoType := common.RepoTypeFromContext(ctx)
+ if repoType == types.UnknownRepo {
+ slog.Error("Bad request of repo type")
+ httpbase.BadRequest(ctx, "Bad request of repo type")
+ return
+ }
+ response, err := h.c.ListRuntimeFrameworkWithType(ctx, deployType)
+ if err != nil {
+ slog.Error("fail to list runtime framework", slog.String("error", err.Error()))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, response)
+}
+
+// serverlessDetail godoc
+// @Security ApiKey
+// @Summary Get repo serverless detail
+// @Description Get repo serverless detail
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "id"
+// @Param current_user query string false "current user"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless/{id} [get]
+func (h *RepoHandler) ServerlessDetail(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ detailReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.ServerlessType,
+ }
+
+ response, err := h.c.DeployDetail(ctx, detailReq)
+ if err != nil {
+ slog.Error("fail to serverless detail", slog.String("error", err.Error()), slog.Any("namespace", namespace), slog.Any("name", name), slog.Any("deploy id", deployID))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, response)
+}
+
+// GetServerlessLogs godoc
+// @Security ApiKey
+// @Summary get serverless logs
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param repo_type path string true "models" Enums(models)
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "id"
+// @Param instance path string true "instance"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless/{id}/logs/{instance} [get]
+func (h *RepoHandler) ServerlessLogs(ctx *gin.Context) {
+ if ctx.Query("test") == "true" {
+ h.testLogs(ctx)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace and name from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ repoType := common.RepoTypeFromContext(ctx)
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ instance := ctx.Param("instance")
+ if len(instance) < 1 {
+ httpbase.UnauthorizedError(ctx, errors.New("fail to get deploy instance"))
+ return
+ }
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ logReq := types.DeployActReq{
+ RepoType: repoType,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.ServerlessType,
+ InstanceName: instance,
+ }
+
+ // user http request context instead of gin context, so that server knows the life cycle of the request
+ logReader, err := h.c.DeployInstanceLogs(ctx.Request.Context(), logReq)
+ if err != nil {
+ var pErr *types.PermissionError
+ if errors.As(err, &pErr) {
+ httpbase.UnauthorizedError(ctx, err)
+ } else {
+ slog.Error("Failed to get deploy instance logs", slog.Any("logReq", logReq), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ }
+ return
+ }
+
+ if logReader.RunLog() == nil {
+ httpbase.ServerError(ctx, errors.New("don't find any deploy instance log"))
+ return
+ }
+
+ // to quickly respond the http request
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("repo handler logs request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ case data, ok := <-logReader.RunLog():
+ if ok {
+ ctx.SSEvent("Container", string(data))
+ ctx.Writer.Flush()
+ }
+ }
+ }
+}
+
+// GetServerlessStatus godoc
+// @Security JWT token
+// @Summary get serverless status
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "deploy id"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless/{id}/status [get]
+func (h *RepoHandler) ServerlessStatus(ctx *gin.Context) {
+ if ctx.Query("test") == "true" {
+ h.testStatus(ctx)
+ return
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ statusReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.ServerlessType,
+ }
+
+ allow, err := h.c.AllowAccessDeploy(ctx, statusReq)
+ if err != nil {
+ slog.Error("failed to check user permission", slog.Any("error", err))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to check user permission, %w", err))
+ return
+ }
+
+ if !allow {
+ slog.Info("user not allowed to query deploy status", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser), slog.Any("deploy_id", deployID))
+ }
+
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("deploy handler status request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ default:
+ time.Sleep(time.Second * 5)
+ // user http request context instead of gin context, so that server knows the life cycle of the request
+ _, status, instances, err := h.c.DeployStatus(ctx.Request.Context(), types.ModelRepo, namespace, name, deployID)
+ if err != nil {
+ slog.Error("failed to get deploy status", slog.Any("error", err), slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("deploy_id", deployID))
+ ctx.SSEvent("error", err.Error())
+ } else {
+ eventData := &types.ModelStatusEventData{
+ Status: status,
+ Details: instances,
+ }
+ ctx.SSEvent("status", eventData)
+
+ }
+ ctx.Writer.Flush()
+ }
+ }
+}
+
+// ServerlessUpdate godoc
+// @Security ApiKey
+// @Summary Update serverless parameters
+// @Tags Model
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param id path string true "deploy id"
+// @Param current_user query string true "current_user"
+// @Param body body types.DeployUpdateReq true "deploy setting of Serverless"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /models/{namespace}/{name}/serverless/{id} [put]
+func (h *RepoHandler) ServerlessUpdate(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace and name from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req *types.DeployUpdateReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err, slog.Any("request.body", ctx.Request.Body))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ if req.MinReplica != nil && req.MaxReplica != nil {
+ err = Validate.Struct(req)
+ if err != nil {
+ slog.Error("Bad request setting for serverless", slog.Any("req", *req), slog.Any("err", err))
+ httpbase.BadRequest(ctx, fmt.Sprintf("Bad request setting for serverless, %v", err))
+ return
+ }
+ }
+
+ deployID, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", slog.Any("error", err), slog.Any("id", ctx.Param("id")))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ updateReq := types.DeployActReq{
+ RepoType: types.ModelRepo,
+ Namespace: namespace,
+ Name: name,
+ CurrentUser: currentUser,
+ DeployID: deployID,
+ DeployType: types.ServerlessType,
+ }
+ err = h.c.DeployUpdate(ctx, updateReq, req)
+ if err != nil {
+ slog.Error("failed to update serverless", slog.String("namespace", namespace), slog.String("name", name), slog.Any("username", currentUser), slog.Int64("deploy_id", deployID), slog.Any("error", err))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to update serverless, %w", err))
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/proxy"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+type RProxyHandler struct {
+ SpaceRootDomain string
+ spaceComp component.SpaceComponent
+ repoComp component.RepoComponent
+}
+
+func NewRProxyHandler(config *config.Config) (*RProxyHandler, error) {
+ spaceComp, err := component.NewSpaceComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create space component,%w", err)
+ }
+ repoComp, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create repo component,%w", err)
+ }
+
+ return &RProxyHandler{
+ SpaceRootDomain: config.Space.InternalRootDomain,
+ spaceComp: spaceComp,
+ repoComp: repoComp,
+ }, nil
+}
+
+func (r *RProxyHandler) Proxy(ctx *gin.Context) {
+ slog.Debug("http request", slog.Any("request", ctx.Request.URL), slog.Any("header", ctx.Request.Header))
+ appSrvName := r.GetSrvName(ctx)
+
+ deploy, err := r.repoComp.GetDeployBySvcName(ctx, appSrvName)
+ if err != nil {
+ slog.Error("failed to get deploy in rproxy", slog.Any("error", err), slog.Any("appSrvName", appSrvName))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to get deploy, %w", err))
+ return
+ }
+
+ username := httpbase.GetCurrentUser(ctx)
+ allow := false
+ err = nil
+ if deploy.SpaceID > 0 {
+ // user must login to visit space
+ if httpbase.GetAuthType(ctx) != httpbase.AuthTypeJwt {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found in session, please access with jwt token first"))
+ return
+ }
+
+ // check space
+ allow, err = r.repoComp.AllowAccessByRepoID(ctx, deploy.RepoID, username)
+ } else if deploy.ModelID > 0 {
+ // check model inference
+ allow, err = r.repoComp.AllowAccessEndpoint(ctx, username, deploy)
+ }
+
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, fmt.Errorf("failed to check user permission,%w", err))
+ return
+ }
+
+ if allow {
+ apiname := ctx.Param("api")
+ target := fmt.Sprintf("http://%s.%s", appSrvName, r.SpaceRootDomain)
+ if deploy.Endpoint != "" {
+ //support multi-cluster
+ target = deploy.Endpoint
+ }
+ rp, _ := proxy.NewReverseProxy(target)
+ if deploy.Type == 1 || deploy.Type == 3 {
+ //for infernece,no need context path
+ contextPath := fmt.Sprintf("/%s/%s", "endpoint", appSrvName)
+ apiname = strings.TrimPrefix(apiname, contextPath)
+ }
+ rp.ServeHTTP(ctx.Writer, ctx.Request, apiname)
+ } else {
+ slog.Warn("user not allowed to call endpoint api", slog.String("srv_name", appSrvName), slog.Any("user_name", username), slog.Any("deployID", deploy.ID))
+ ctx.Status(http.StatusForbidden)
+ }
+}
+
+// get service name based on request
+func (r *RProxyHandler) GetSrvName(ctx *gin.Context) string {
+ URI := ctx.Request.RequestURI
+ host := ctx.Request.Host
+ //check if request is from internal endpoint
+ if strings.HasPrefix(URI, "/endpoint/") {
+ //for case: http://127.0.0.1:8080/endpoint/dx1jpfny9hq8
+ parts := strings.SplitN(ctx.Request.URL.Path, "/", 5)
+ return parts[2]
+ } else {
+ // for case: https://dx1jpfny9hq8.cn-beijing.aliyun.space.opencsg.com
+ domainParts := strings.SplitN(host, ".", 2)
+ appSrvName := domainParts[0]
+ return appSrvName
+ }
+
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewRuntimeArchitectureHandler(config *config.Config) (*RuntimeArchitectureHandler, error) {
+ nrc, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create repo component, %w", err)
+ }
+ nrac, err := component.NewRuntimeArchitectureComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create runtime arch component, %w", err)
+ }
+
+ return &RuntimeArchitectureHandler{
+ rc: nrc,
+ rac: nrac,
+ }, nil
+}
+
+type RuntimeArchitectureHandler struct {
+ rc component.RepoComponent
+ rac component.RuntimeArchitectureComponent
+}
+
+// GetArchitectures godoc
+// @Security ApiKey
+// @Summary Get runtime framework architectures
+// @Description get runtime framework architectures
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id}/architecture [get]
+func (r *RuntimeArchitectureHandler) ListByRuntimeFrameworkID(ctx *gin.Context) {
+ strID := ctx.Param("id")
+ id, err := strconv.ParseInt(strID, 10, 64)
+ if err != nil {
+ slog.Error("invalid runtime framework ID", slog.Any("error", err))
+ httpbase.BadRequest(ctx, "invalid runtime framework ID format")
+ return
+ }
+ resp, err := r.rac.ListByRuntimeFrameworkID(ctx, id)
+ if err != nil {
+ slog.Error("fail to list runtime architectures", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, resp)
+}
+
+// UpdateArchitectures godoc
+// @Security ApiKey
+// @Summary Set runtime framework architectures
+// @Description set runtime framework architectures
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Param body body types.RuntimeArchitecture true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id}/architecture [put]
+func (r *RuntimeArchitectureHandler) UpdateArchitecture(ctx *gin.Context) {
+ var req types.RuntimeArchitecture
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request runtime framework id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ res, err := r.rac.SetArchitectures(ctx, id, req.Architectures)
+ if err != nil {
+ slog.Error("Failed to set architectures", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, res)
+}
+
+// DeleteArchitectures godoc
+// @Security ApiKey
+// @Summary Delete runtime framework architectures
+// @Description Delete runtime framework architectures
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Param body body types.RuntimeArchitecture true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id}/architecture [delete]
+func (r *RuntimeArchitectureHandler) DeleteArchitecture(ctx *gin.Context) {
+ var req types.RuntimeArchitecture
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request runtime framework id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ list, err := r.rac.DeleteArchitectures(ctx, id, req.Architectures)
+ if err != nil {
+ slog.Error("Failed to delete architectures", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, list)
+}
+
+// ScanArchitecture godoc
+// @Security ApiKey
+// @Summary Scan runtime architecture
+// @Description Scan runtime architecture
+// @Tags RuntimeFramework
+// @Accept json
+// @Produce json
+// @Param id path int true "runtime framework id"
+// @Param scan_type query int false "scan_type(0:all models, 1:new models, 2:old models)" Enums(0, 1, 2)
+// @Param body body types.RuntimeFrameworkModels true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /runtime_framework/{id}/scan [post]
+func (r *RuntimeArchitectureHandler) ScanArchitecture(ctx *gin.Context) {
+ id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request runtime framework id format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ scanTypeStr := ctx.Query("scan_type")
+ if scanTypeStr == "" {
+ slog.Error("Bad request scan type")
+ httpbase.BadRequest(ctx, "bad request scan type")
+ return
+ }
+ scanType, err := strconv.Atoi(scanTypeStr)
+ if err != nil {
+ slog.Error("Bad request scan format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.RuntimeFrameworkModels
+ err = ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Failed to bind json", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = r.rac.ScanArchitecture(ctx, id, scanType, req.Models)
+ if err != nil {
+ slog.Error("Failed to scan architecture", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+type SensitiveHandler struct {
+ c component.SensitiveComponent
+}
+
+func NewSensitiveHandler(cfg *config.Config) (*SensitiveHandler, error) {
+ sc, err := component.NewSensitiveComponent(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &SensitiveHandler{
+ c: sc,
+ }, nil
+}
+
+func (h *SensitiveHandler) Text(ctx *gin.Context) {
+ type req struct {
+ Scenario string `json:"scenario"`
+ Text string `json:"text"`
+ }
+ var (
+ r req
+ err error
+ )
+ if err = ctx.ShouldBindJSON(&r); err != nil {
+ slog.Error("Bad request format", slog.String("err", err.Error()))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ok, err := h.c.CheckText(ctx, r.Scenario, r.Text)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ if ok {
+ httpbase.OK(ctx, nil)
+ } else {
+ httpbase.BadRequest(ctx, "sensitive content detected")
+ }
+
+}
+
+func (h *SensitiveHandler) Image(ctx *gin.Context) {
+ type req struct {
+ Scenario string `json:"scenario"`
+ OssBucketName string `json:"oss_bucket_name"`
+ OssObjectName string `json:"oss_object_name"`
+ }
+ var (
+ r req
+ err error
+ )
+ if err = ctx.ShouldBindJSON(&r); err != nil {
+ slog.Error("Bad request format", slog.String("err", err.Error()))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ ok, err := h.c.CheckImage(ctx, r.Scenario, r.OssBucketName, r.OssObjectName)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ if ok {
+ httpbase.OK(ctx, nil)
+ } else {
+ httpbase.BadRequest(ctx, "sensitive content detected")
+ }
+
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "slices"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewSpaceHandler(config *config.Config) (*SpaceHandler, error) {
+ c, err := component.NewSpaceComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ ssc, err := component.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ repo, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating repo component:%w", err)
+ }
+ return &SpaceHandler{
+ c: c,
+ ssc: ssc,
+ repo: repo,
+ }, nil
+}
+
+type SpaceHandler struct {
+ c component.SpaceComponent
+ ssc component.SensitiveComponent
+ repo component.RepoComponent
+}
+
+// GetAllSpaces godoc
+// @Security ApiKey
+// @Summary Get spaces visible to current user
+// @Description get spaces visible to current user
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param current_user query string false "current user"
+// @Param search query string false "search text"
+// @Param task_tag query string false "filter by task tag"
+// @Param framework_tag query string false "filter by framework tag"
+// @Param license_tag query string false "filter by license tag"
+// @Param language_tag query string false "filter by language tag"
+// @Param sort query string false "sort by"
+// @Param source query string false "source" Enums(opencsg, huggingface, local)
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Space,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces [get]
+func (h *SpaceHandler) Index(ctx *gin.Context) {
+ filter := new(types.RepoFilter)
+ filter.Tags = parseTagReqs(ctx)
+ filter.Username = httpbase.GetCurrentUser(ctx)
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ filter = getFilterFromContext(ctx, filter)
+ if !slices.Contains[[]string](Sorts, filter.Sort) {
+ msg := fmt.Sprintf("sort parameter must be one of %v", Sorts)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ httpbase.BadRequest(ctx, msg)
+ return
+ }
+
+ if filter.Source != "" && !slices.Contains[[]string](Sources, filter.Source) {
+ msg := fmt.Sprintf("source parameter must be one of %v", Sources)
+ slog.Error("Bad request format,", slog.String("error", msg))
+ ctx.JSON(http.StatusBadRequest, gin.H{"message": msg})
+ return
+ }
+
+ spaces, total, err := h.c.Index(ctx, filter, per, page)
+ if err != nil {
+ slog.Error("Failed to get spaces", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get public spaces succeed", slog.Int("count", total))
+ respData := gin.H{
+ "data": spaces,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// ShowSpaceDetail godoc
+// @Security ApiKey
+// @Summary show space detail
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{data=types.Space} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name} [get]
+func (h *SpaceHandler) Show(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ detail, err := h.c.Show(ctx, namespace, name, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to get space detail", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, detail)
+}
+
+// CreateSpace godoc
+// @Security ApiKey
+// @Summary Create a new space
+// @Description create a new space
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param current_user query string true "current_user"
+// @Param body body types.CreateSpaceReq true "body"
+// @Success 200 {object} types.Response{data=types.Space} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces [post]
+func (h *SpaceHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.CreateSpaceReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err := h.ssc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ space, err := h.c.Create(ctx, req)
+ if err != nil {
+ slog.Error("Failed to create space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create space succeed", slog.String("space", space.Name))
+ httpbase.OK(ctx, space)
+}
+
+// UpdateSpace godoc
+// @Security ApiKey
+// @Summary Update a exists space
+// @Description update a exists space
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Param body body types.UpdateSpaceReq true "body"
+// @Success 200 {object} types.Response{data=types.Space} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name} [put]
+func (h *SpaceHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req *types.UpdateSpaceReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ _, err := h.ssc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.Username = currentUser
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.Namespace = namespace
+ req.Name = name
+
+ space, err := h.c.Update(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Update space succeed", slog.String("space", space.Name))
+ httpbase.OK(ctx, space)
+}
+
+// DeleteSpace godoc
+// @Security ApiKey
+// @Summary Delete a exists space
+// @Description delete a exists space
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name} [delete]
+func (h *SpaceHandler) Delete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.Delete(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("Failed to delete space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Delete space succeed", slog.String("space", name))
+ httpbase.OK(ctx, nil)
+}
+
+// RunSpace godoc
+// @Security JWT token
+// @Summary run space app
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name}/run [post]
+func (h *SpaceHandler) Run(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ allow, err := h.repo.AllowAdminAccess(ctx, types.SpaceRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+ if !allow {
+ slog.Info("user not allowed to run space", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ httpbase.UnauthorizedError(ctx, errors.New("user not allowed to run sapce"))
+ return
+ }
+ deployID, err := h.c.Deploy(ctx, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to deploy space", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New("failed to deploy space"))
+ return
+ }
+
+ slog.Info("space deployment created", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Int64("deploy_id", deployID))
+ httpbase.OK(ctx, nil)
+}
+
+// WakeupSpace godoc
+// @Security JWT token
+// @Summary wake up space app
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name}/wakeup [post]
+func (h *SpaceHandler) Wakeup(ctx *gin.Context) {
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.Wakeup(ctx, namespace, name)
+ if err != nil {
+ slog.Error("failed to wakeup space", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New("failed to wakeup space"))
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// StopSpace godoc
+// @Security JWT token
+// @Summary stop space app
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name}/stop [post]
+func (h *SpaceHandler) Stop(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ allow, err := h.repo.AllowAdminAccess(ctx, types.SpaceRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+ if !allow {
+ slog.Info("user not allowed to stop space", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ httpbase.UnauthorizedError(ctx, errors.New("user not allowed to stop sapce"))
+ return
+ }
+
+ err = h.c.Stop(ctx, namespace, name, false)
+ if err != nil {
+ slog.Error("failed to stop space", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New("failed to stop space"))
+ return
+ }
+
+ slog.Info("stop space success", slog.String("namespace", namespace),
+ slog.String("name", name))
+ httpbase.OK(ctx, nil)
+}
+
+// GetSpaceStatus godoc
+// @Security JWT token
+// @Summary get space status
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name}/status [get]
+func (h *SpaceHandler) Status(ctx *gin.Context) {
+ if ctx.Query("test") == "true" {
+ h.status(ctx)
+ return
+ }
+
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ allow, err := h.repo.AllowReadAccess(ctx, types.SpaceRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+
+ if !allow {
+ slog.Info("user not allowed to query space status", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ }
+
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("space handler status request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ default:
+ time.Sleep(time.Second * 5)
+ //user http request context instead of gin context, so that server knows the life cycle of the request
+ _, status, err := h.c.Status(ctx.Request.Context(), namespace, name)
+ if err != nil {
+ slog.Error("failed to get space status", slog.Any("error", err), slog.String("namespace", namespace),
+ slog.String("name", name))
+ ctx.SSEvent("error", err.Error())
+ } else {
+ ctx.SSEvent("status", status)
+ }
+ ctx.Writer.Flush()
+ }
+ }
+}
+
+func (h *SpaceHandler) status(ctx *gin.Context) {
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("space handler status request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ default:
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Building")
+ ctx.Writer.Flush()
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Running")
+ ctx.Writer.Flush()
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Sleeping")
+ ctx.Writer.Flush()
+ time.Sleep(time.Second * 5)
+ ctx.SSEvent("status", "Stopped")
+ ctx.Writer.Flush()
+ }
+ }
+}
+
+// GetSpaceLogs godoc
+// @Security JWT token
+// @Summary get space logs
+// @Tags Space
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param name path string true "name"
+// @Param current_user query string true "current_user"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /spaces/{namespace}/{name}/logs [get]
+func (h *SpaceHandler) Logs(ctx *gin.Context) {
+ if ctx.Query("test") == "true" {
+ h.testLogs(ctx)
+ return
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromContext(ctx)
+ if err != nil {
+ slog.Error("failed to get namespace from context", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ currentUser := httpbase.GetCurrentUser(ctx)
+ allow, err := h.repo.AllowReadAccess(ctx, types.SpaceRepo, namespace, name, currentUser)
+ if err != nil {
+ slog.Error("failed to check user permission", "error", err)
+ httpbase.ServerError(ctx, errors.New("failed to check user permission"))
+ return
+ }
+
+ if !allow {
+ slog.Info("user not allowed to read sapce logs", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("username", currentUser))
+ httpbase.UnauthorizedError(ctx, errors.New("user not allowed to read sapce logs"))
+ return
+ }
+
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ //user http request context instead of gin context, so that server knows the life cycle of the request
+ logReader, err := h.c.Logs(ctx.Request.Context(), namespace, name)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ if logReader.RunLog() == nil && logReader.BuildLog() == nil {
+ httpbase.ServerError(ctx, errors.New("don't find any space deploy log"))
+ return
+ }
+
+ //to quickly respond the http request
+ ctx.Writer.WriteHeader(http.StatusOK)
+ ctx.Writer.Flush()
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ slog.Info("space handler logs request context done", slog.Any("error", ctx.Request.Context().Err()))
+ return
+ case data, ok := <-logReader.BuildLog():
+ if ok {
+ ctx.SSEvent("Build", string(data))
+ ctx.Writer.Flush()
+ }
+ case data, ok := <-logReader.RunLog():
+ if ok {
+ ctx.SSEvent("Container", string(data))
+ ctx.Writer.Flush()
+ }
+ }
+ }
+}
+
+func (h *SpaceHandler) testLogs(ctx *gin.Context) {
+ ctx.Writer.Header().Set("Content-Type", "text/event-stream")
+ ctx.Writer.Header().Set("Cache-Control", "no-cache")
+ ctx.Writer.Header().Set("Connection", "keep-alive")
+ ctx.Writer.Header().Set("Transfer-Encoding", "chunked")
+
+ for {
+ select {
+ case <-ctx.Request.Context().Done():
+ return
+ default:
+ ctx.SSEvent("Build", "test build log message")
+ ctx.SSEvent("Container", "test run log message")
+ ctx.Writer.Flush()
+ }
+ time.Sleep(time.Second * 5)
+ }
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewSpaceResourceHandler(config *config.Config) (*SpaceResourceHandler, error) {
+ src, err := component.NewSpaceResourceComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &SpaceResourceHandler{
+ c: src,
+ }, nil
+}
+
+type SpaceResourceHandler struct {
+ c component.SpaceResourceComponent
+}
+
+// GetSpaceResources godoc
+// @Security ApiKey
+// @Summary Get space resources
+// @Description get space resources
+// @Tags SpaceReource
+// @Accept json
+// @Produce json
+// @Param cluster_id query string false "cluster_id"
+// @Param deploy_type query int false "deploy type(0-space,1-inference,2-finetune)" Enums(0, 1, 2) default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.SpaceResource,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_resources [get]
+func (h *SpaceResourceHandler) Index(ctx *gin.Context) {
+ clusterId := ctx.Query("cluster_id")
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ spaceResources, err := h.c.Index(ctx, clusterId, deployType, currentUser)
+ if err != nil {
+ slog.Error("Failed to get space resources", slog.String("cluster_id", clusterId), slog.String("deploy_type", deployTypeStr), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get space resources successfully")
+ httpbase.OK(ctx, spaceResources)
+}
+
+// CreateSpaceResource godoc
+// @Security ApiKey
+// @Summary Create space resource
+// @Description create space resource
+// @Tags SpaceReource
+// @Accept json
+// @Produce json
+// @Param body body types.CreateSpaceResourceReq true "body"
+// @Success 200 {object} types.ResponseWithTotal{data=types.SpaceResource,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_resources [post]
+func (h *SpaceResourceHandler) Create(ctx *gin.Context) {
+ var req types.CreateSpaceResourceReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ spaceResource, err := h.c.Create(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create space resources", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create space resources successfully")
+ httpbase.OK(ctx, spaceResource)
+}
+
+// UpdateSpaceResource godoc
+// @Security ApiKey
+// @Summary Update a exist space resource
+// @Description update a exist space resource
+// @Tags SpaceReource
+// @Accept json
+// @Produce json
+// @Param id path int true "id"
+// @Param body body types.UpdateSpaceResourceReq true "body"
+// @Success 200 {object} types.ResponseWithTotal{data=types.SpaceResource,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_resources/{id} [put]
+func (h *SpaceResourceHandler) Update(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ var req *types.UpdateSpaceResourceReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.ID = id
+
+ spaceResource, err := h.c.Update(ctx, req)
+ if err != nil {
+ slog.Error("Failed to update space resource", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Update space resources successfully")
+ httpbase.OK(ctx, spaceResource)
+}
+
+// DeleteSpaceResource godoc
+// @Security ApiKey
+// @Summary Delete a exist space resource
+// @Description delete a exist space resource
+// @Tags SpaceReource
+// @Accept json
+// @Produce json
+// @Param id path int true "id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_resources/{id} [delete]
+func (h *SpaceResourceHandler) Delete(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = h.c.Delete(ctx, id)
+ if err != nil {
+ slog.Error("Failed to delete space resource", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Delete space resource successfully")
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewSpaceSdkHandler(config *config.Config) (*SpaceSdkHandler, error) {
+ ssc, err := component.NewSpaceSdkComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &SpaceSdkHandler{
+ c: ssc,
+ }, nil
+}
+
+type SpaceSdkHandler struct {
+ c component.SpaceSdkComponent
+}
+
+// GetSpaceSdks godoc
+// @Security ApiKey
+// @Summary Get space sdks
+// @Description get space sdks
+// @Tags SpaceSdk
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.SpaceSdk,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_sdks [get]
+func (h *SpaceSdkHandler) Index(ctx *gin.Context) {
+ spaceSdks, err := h.c.Index(ctx)
+ if err != nil {
+ slog.Error("Failed to get space sdks", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Get space sdks successfully")
+ httpbase.OK(ctx, spaceSdks)
+}
+
+// CreateSpaceSdk godoc
+// @Security ApiKey
+// @Summary Create space sdk
+// @Description create space sdk
+// @Tags SpaceSdk
+// @Accept json
+// @Produce json
+// @Param body body types.CreateSpaceSdkReq true "body"
+// @Success 200 {object} types.ResponseWithTotal{data=types.SpaceSdk,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_sdks [post]
+func (h *SpaceSdkHandler) Create(ctx *gin.Context) {
+ var req types.CreateSpaceSdkReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ spaceSdk, err := h.c.Create(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create space sdk", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Create space sdks successfully")
+ httpbase.OK(ctx, spaceSdk)
+}
+
+// UpdateSpaceSdk godoc
+// @Security ApiKey
+// @Summary Update a exist space sdk
+// @Description update a exist space sdk
+// @Tags SpaceSdk
+// @Accept json
+// @Produce json
+// @Param id path int true "id"
+// @Param body body types.UpdateSpaceSdkReq true "body"
+// @Success 200 {object} types.ResponseWithTotal{data=types.SpaceSdk,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_sdks/{id} [put]
+func (h *SpaceSdkHandler) Update(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ var req types.UpdateSpaceSdkReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.ID = id
+
+ spaceSdk, err := h.c.Update(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to update space sdk", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Update space sdks successfully")
+ httpbase.OK(ctx, spaceSdk)
+}
+
+// DeleteSpaceSdk godoc
+// @Security ApiKey
+// @Summary Delete a exist space sdk
+// @Description delete a exist space sdk
+// @Tags SpaceSdk
+// @Accept json
+// @Produce json
+// @Param 写id path int true "id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /space_sdks/{id} [delete]
+func (h *SpaceSdkHandler) Delete(ctx *gin.Context) {
+ var (
+ id int64
+ err error
+ )
+ id, err = strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ err = h.c.Delete(ctx, id)
+ if err != nil {
+ slog.Error("Failed to delete space sdk", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ slog.Info("Delete space sdk successfully")
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewSSHKeyHandler(config *config.Config) (*SSHKeyHandler, error) {
+ oc, err := component.NewSSHKeyComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := component.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &SSHKeyHandler{
+ c: oc,
+ sc: sc,
+ }, nil
+}
+
+type SSHKeyHandler struct {
+ c component.SSHKeyComponent
+ sc component.SensitiveComponent
+}
+
+// CreateUserSSHKey godoc
+// @Security ApiKey
+// @Summary Create a new SSH key for the given user
+// @Description create a new SSH key for the given user
+// @Tags SSH Key
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @param body body types.CreateSSHKeyRequest true "body"
+// @Success 200 {object} types.Response{data=database.SSHKey} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/ssh_keys [post]
+func (h *SSHKeyHandler) Create(ctx *gin.Context) {
+ var req types.CreateSSHKeyRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ _, err := h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ req.Username = currentUser
+ sk, err := h.c.Create(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create SSH key", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Create SSH key succeed", slog.String("key_name", sk.Name))
+ httpbase.OK(ctx, sk)
+}
+
+// GetUserSSHKeys godoc
+// @Security ApiKey
+// @Summary Get all SSH keys for the given user
+// @Description get all SSH keys for the given user
+// @Tags SSH Key
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]database.SSHKey,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/ssh_keys [get]
+func (h *SSHKeyHandler) Index(ctx *gin.Context) {
+ username := ctx.Param("username")
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ sks, err := h.c.Index(ctx, username, per, page)
+ if err != nil {
+ slog.Error("Failed to create SSH key", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get User SSH keys succeed")
+ httpbase.OK(ctx, sks)
+}
+
+// DeleteUserSSHKey godoc
+// @Security ApiKey
+// @Summary Delete specific SSH key for the given user
+// @Description delete specific SSH key for the given user
+// @Tags SSH Key
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param name path string true "key name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/ssh_key/{name} [delete]
+func (h *SSHKeyHandler) Delete(ctx *gin.Context) {
+ name := ctx.Param("name")
+ username := ctx.Param("username")
+ if name == "" || username == "" {
+ err := fmt.Errorf("invalid username or key name in url")
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err := h.c.Delete(ctx, username, name)
+ if err != nil {
+ slog.Error("Failed to delete SSH key", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Delete SSH keys succeed")
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+type SyncClientSettingHandler struct {
+ c component.SyncClientSettingComponent
+}
+
+func NewSyncClientSettingHandler(config *config.Config) (*SyncClientSettingHandler, error) {
+ c, err := component.NewSyncClientSettingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &SyncClientSettingHandler{
+ c: c,
+ }, nil
+}
+
+// CreateSyncClientSetting godoc
+// @Security ApiKey
+// @Summary Create sync client setting or update an existing sync client setting
+// @Description Create sync client setting or update an existing sync client setting
+// @Tags Sync
+// @Accept json
+// @Produce json
+// @Param body body types.CreateSyncClientSettingReq true "body"
+// @Success 200 {object} types.Response{data=database.SyncClientSetting} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /sync/client_setting [post]
+func (h *SyncClientSettingHandler) Create(ctx *gin.Context) {
+ var req types.CreateSyncClientSettingReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ if req.CurrentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ ms, err := h.c.Create(ctx, req)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to create sync client setting", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ms)
+}
+
+// GetSyncClientSetting godoc
+// @Security ApiKey
+// @Summary Get sync client setting
+// @Description Get sync client setting
+// @Tags Sync
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{data=database.SyncClientSetting} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /sync/client_setting [get]
+func (h *SyncClientSettingHandler) Show(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ ms, err := h.c.Show(ctx, currentUser)
+ if err != nil {
+ if errors.Is(err, component.ErrUnauthorized) {
+ httpbase.UnauthorizedError(ctx, err)
+ return
+ }
+ slog.Error("Failed to find sync client setting", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ms)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewTagHandler(config *config.Config) (*TagsHandler, error) {
+ tc, err := component.NewTagComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &TagsHandler{
+ tc: tc,
+ }, nil
+}
+
+type TagsHandler struct {
+ tc component.TagComponent
+}
+
+// GetAllTags godoc
+// @Security ApiKey
+// @Summary Get all tags
+// @Description get all tags
+// @Tags Tag
+// @Accept json
+// @Produce json
+// @Param category query string false "category name"
+// @Param scope query string false "scope name" Enums(model, dataset)
+// @Success 200 {object} types.ResponseWithTotal{data=[]database.Tag,total=int} "tags"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /tags [get]
+func (t *TagsHandler) AllTags(ctx *gin.Context) {
+ //TODO:validate inputs
+ category := ctx.Query("category")
+ scope := ctx.Query("scope")
+ tags, err := t.tc.AllTagsByScopeAndCategory(ctx, scope, category)
+ if err != nil {
+ slog.Error("Failed to load tags", "error", err)
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": tags,
+ }
+
+ slog.Info("Tags loaded successfully", "count", len(tags))
+ ctx.JSON(http.StatusOK, respData)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types/telemetry"
+ "opencsg.com/csghub-server/component"
+)
+
+type TelemetryHandler struct {
+ c component.TelemetryComponent
+}
+
+func NewTelemetryHandler() (*TelemetryHandler, error) {
+ c, err := component.NewTelemetryComponent()
+ if err != nil {
+ return nil, fmt.Errorf("fail to create TelemetryComponent,%w", err)
+ }
+ return &TelemetryHandler{
+ c: c,
+ }, nil
+}
+
+// Usage godoc
+// @Security ApiKey
+// @Summary Submit telemetry data for a client
+// @Tags Telemetry
+// @Accept json
+// @Produce json
+// @Param body body telemetry.Usage true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /telemetry/usage [post]
+func (th *TelemetryHandler) Usage(ctx *gin.Context) {
+ var usage telemetry.Usage
+ if err := ctx.ShouldBindJSON(&usage); err != nil {
+ newErr := fmt.Errorf("bad request format, %w", err).Error()
+ slog.Error(newErr)
+ httpbase.BadRequest(ctx, newErr)
+ return
+ }
+ err := th.c.SaveUsageData(ctx.Request.Context(), usage)
+ if err != nil {
+ slog.Error("fail to save usage data", slog.Any("error", err))
+ httpbase.ServerError(ctx, errors.New("fail to save usage data"))
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "log/slog"
+ "net/http"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/component"
+)
+
+func NewUserHandler(config *config.Config) (*UserHandler, error) {
+ uc, err := component.NewUserComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &UserHandler{
+ c: uc,
+ }, nil
+}
+
+type UserHandler struct {
+ c component.UserComponent
+}
+
+// GetUserDatasets godoc
+// @Security ApiKey
+// @Summary Get user datasets
+// @Description get user datasets
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Dataset,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/datasets [get]
+func (h *UserHandler) Datasets(ctx *gin.Context) {
+ var req types.UserDatasetsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ds, total, err := h.c.Datasets(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user datasets", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user datasets succeed", slog.String("user", req.Owner))
+ respData := gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserModels godoc
+// @Security ApiKey
+// @Summary Get user models
+// @Description get user models
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Model,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/models [get]
+func (h *UserHandler) Models(ctx *gin.Context) {
+ var req types.UserDatasetsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.Models(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user models", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user models succeed", slog.String("user", req.Owner))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ms,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserCodes godoc
+// @Security ApiKey
+// @Summary Get user codes
+// @Description get user codes
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Code,total=int} "OK"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Router /user/{username}/codes [get]
+func (h *UserHandler) Codes(ctx *gin.Context) {
+ var req types.UserDatasetsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.Codes(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user codes", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user codes succeed", slog.String("user", req.Owner))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ms,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserSpaces godoc
+// @Security ApiKey
+// @Summary Get user spaces
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Space,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/spaces [get]
+func (h *UserHandler) Spaces(ctx *gin.Context) {
+ var req types.UserSpacesReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.Spaces(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ms,
+ "total": total,
+ }
+
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// AddUserLikes godoc
+// @Security ApiKey
+// @Summary Add user likes
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param repo_id path string true "repo id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/{repoid} [put]
+func (h *UserHandler) LikesAdd(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.UserLikesRequest
+ req.Username = ctx.Param("username")
+ req.CurrentUser = currentUser
+ repo_id, err := strconv.ParseInt(ctx.Param("repo_id"), 10, 64)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ req.Repo_id = repo_id
+ err = h.c.AddLikes(ctx, &req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetLikesCollections godoc
+// @Security ApiKey
+// @Summary Get user likes collections
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Collection,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/collections [get]
+func (h *UserHandler) LikesCollections(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req types.UserCollectionReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.LikesCollection(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get user collections", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": ms,
+ "total": total,
+ }
+
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserCollections godoc
+// @Security ApiKey
+// @Summary Get user's collections
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Collection,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/collections [get]
+func (h *UserHandler) UserCollections(ctx *gin.Context) {
+ var req types.UserCollectionReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.Collections(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get user collections", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": ms,
+ "total": total,
+ }
+
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// AddCollectionLikes godoc
+// @Security ApiKey
+// @Summary Add collection likes
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param id path string true "collection id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/collections/{id} [put]
+func (h *UserHandler) LikeCollection(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req types.UserLikesRequest
+ req.CurrentUser = currentUser
+ collection_id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ req.Collection_id = collection_id
+ err = h.c.LikeCollection(ctx, &req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteCollectionLikes godoc
+// @Security ApiKey
+// @Summary delete collection likes
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param id path string true "collection id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/collections/{id} [delete]
+func (h *UserHandler) UnLikeCollection(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req types.UserLikesRequest
+ req.CurrentUser = currentUser
+ collection_id, err := strconv.ParseInt(ctx.Param("id"), 10, 64)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ req.Collection_id = collection_id
+ err = h.c.UnLikeCollection(ctx, &req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteUserlikes godoc
+// @Security ApiKey
+// @Summary Delete user likes
+// @Description Delete user likes
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param repo_id path string true "repo id"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/{repoid} [delete]
+func (h *UserHandler) LikesDelete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.UserLikesRequest
+ req.Username = ctx.Param("username")
+ req.CurrentUser = currentUser
+ repo_id, err := strconv.ParseInt(ctx.Param("repo_id"), 10, 64)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ req.Repo_id = repo_id
+ // slog.Info("user.likes.delete.req=%v", req)
+ err = h.c.DeleteLikes(ctx, &req)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetUserLikesSpaces godoc
+// @Security ApiKey
+// @Summary Get user likes spaces
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Space,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/spaces [get]
+func (h *UserHandler) LikesSpaces(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.UserSpacesReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+
+ ms, total, err := h.c.LikesSpaces(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user space", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ms,
+ "total": total,
+ }
+
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserLikesCodes godoc
+// @Security ApiKey
+// @Summary Get user likes codes
+// @Description get user likes codes
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Code,total=int} "OK"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Router /user/{username}/likes/codes [get]
+func (h *UserHandler) LikesCodes(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.UserDatasetsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.LikesCodes(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user codes", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user likes codes succeed", slog.String("user", req.Owner))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ms,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserLikesModels godoc
+// @Security ApiKey
+// @Summary Get user likes models
+// @Description get user likes models
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Model,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/models [get]
+func (h *UserHandler) LikesModels(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.UserDatasetsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ ms, total, err := h.c.LikesModels(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user models", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user models succeed", slog.String("user", req.Owner))
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ms,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserLikesDatasets godoc
+// @Security ApiKey
+// @Summary Get user likes datasets
+// @Description get user likes datasets
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.Dataset,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/likes/datasets [get]
+func (h *UserHandler) LikesDatasets(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.UserDatasetsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ ds, total, err := h.c.LikesDatasets(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to gat user datasets", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user datasets succeed", slog.String("user", req.Owner))
+ respData := gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+func (h *UserHandler) UserPermission(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ response := types.WhoamiResponse{
+ Name: currentUser,
+ Auth: types.Auth{
+ AccessToken: types.AccessToken{
+ DisplayName: currentUser,
+ Role: "write",
+ },
+ Type: "Bearer",
+ },
+ }
+ ctx.JSON(http.StatusOK, response)
+}
+
+// GetUserRunDeploys godoc
+// @Security ApiKey
+// @Summary Get user running deploys
+// @Description Get user running deploys
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param repo_type path string true "model,space" Enums(model,space)
+// @Param deploy_type query int false "deploy type(0-space,1-inference,2-finetune)" Enums(0, 1, 2) default(1)
+// @Param per query int false "per" default(50)
+// @Param page query int false "page index" default(1)
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/run/{repo_type} [get]
+func (h *UserHandler) GetRunDeploys(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+
+ username := ctx.Param("username")
+ if currentUser != username {
+ slog.Warn("invalid user to list deploys", slog.String("currentUser", currentUser), slog.String("username", username))
+ httpbase.ServerError(ctx, errors.New("invalid user"))
+ return
+ }
+
+ deployTypeStr := ctx.Query("deploy_type")
+ if deployTypeStr == "" {
+ // backward compatibility for inferences
+ deployTypeStr = strconv.Itoa(types.InferenceType)
+ }
+ deployType, err := strconv.Atoi(deployTypeStr)
+ if err != nil {
+ slog.Error("Bad request deploy type format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ repoType := common.RepoTypeFromParam(ctx)
+ if repoType != types.ModelRepo && repoType != types.SpaceRepo {
+ slog.Error("Invalid repo type", slog.Any("repo_type", repoType))
+ httpbase.BadRequest(ctx, "Invalid repo type")
+ return
+ }
+
+ var req types.DeployReq
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ req.RepoType = repoType
+ req.DeployType = deployType
+ ds, total, err := h.c.ListDeploys(ctx, repoType, &req)
+ if err != nil {
+ slog.Error("Failed to get deploy repo list", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetFinetuneInstances godoc
+// @Security ApiKey
+// @Summary Get user running notebook instances
+// @Description Get user running notebook instances
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param per query int false "per" default(50)
+// @Param page query int false "page index" default(1)
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/finetune/instances [get]
+func (h *UserHandler) GetFinetuneInstances(ctx *gin.Context) {
+ respData := gin.H{
+ "message": "OK",
+ "data": nil,
+ "total": 0,
+ }
+
+ var req types.UserRepoReq
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ ctx.JSON(http.StatusOK, respData)
+ return
+ }
+
+ username := ctx.Param("username")
+ if currentUser != username {
+ slog.Warn("invalid user to list deploys", slog.String("currentUser", currentUser), slog.String("username", username))
+ ctx.JSON(http.StatusOK, respData)
+ return
+ }
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ ds, total, err := h.c.ListInstances(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get instance list", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData = gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetServerless godoc
+// @Security ApiKey
+// @Summary Get serverless deploys
+// @Description Get serverless deploys
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param per query int false "per" default(50)
+// @Param page query int false "page index" default(1)
+// @Param current_user query string false "current user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/run/serverless [get]
+func (h *UserHandler) GetRunServerless(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ username := ctx.Param("username")
+ if currentUser != username {
+ slog.Warn("invalid user to list serverless", slog.String("currentUser", currentUser), slog.String("username", username))
+ httpbase.ServerError(ctx, errors.New("invalid user"))
+ return
+ }
+
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var req types.DeployReq
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ req.RepoType = types.ModelRepo
+ req.DeployType = types.ServerlessType
+ ds, total, err := h.c.ListServerless(ctx, req)
+ if err != nil {
+ slog.Error("Failed to get serverless list", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// CreateUserResource godoc
+// @Security ApiKey
+// @Summary create order for user's resource
+// @Description create order for user's resource
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param body body types.CreateUserResourceReq true "create order request"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/order/resource [post]
+func (h *UserHandler) CreateUserResource(ctx *gin.Context) {
+ var req types.CreateUserResourceReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ for i := range req.OrderDetails {
+ req.OrderDetails[i].SkuType = types.SKUCSGHub
+ }
+ req.Username = ctx.Param("username")
+ err := h.c.CreateUserResource(ctx, req)
+ if err != nil {
+ slog.Error("failed to create order for user's resource", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteUserResource godoc
+// @Security ApiKey
+// @Summary delete user's resource by order detail id
+// @Description delete user's resource by order detail id
+// @Tags User
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/order/resources/{id} [delete]
+func (h *UserHandler) DeleteUserResource(ctx *gin.Context) {
+ username := ctx.Param("username")
+ orderDetailIdstr := ctx.Param("id")
+ orderDetailId, err := strconv.ParseInt(orderDetailIdstr, 10, 64)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ err = h.c.DeleteUserResource(ctx, username, orderDetailId)
+ if err != nil {
+ slog.Error("failed to create order for user's resource", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, nil)
+}
+
+// GetUserResource godoc
+// @Security ApiKey
+// @Summary get user's resource
+// @Description get user's resource
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=[]types.UserResourcesResp,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/order/resources [get]
+func (h *UserHandler) GetUserResource(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ username := ctx.Param("username")
+ if currentUser != username {
+ slog.Warn("invalid user to get resources", slog.String("currentUser", currentUser), slog.String("username", username))
+ httpbase.UnauthorizedError(ctx, errors.New("permission denied"))
+ return
+ }
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var req types.GetUserResourceReq
+ req.CurrentUser = currentUser
+ req.Page = page
+ req.PageSize = per
+ ds, total, err := h.c.GetUserResource(ctx, req)
+ if err != nil {
+ slog.Error("failed to get user's resource", slog.Any("error", err), slog.Any("username", username))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserPrompts godoc
+// @Security ApiKey
+// @Summary Get user prompts
+// @Description get user prompts
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/prompts [get]
+func (h *UserHandler) Prompts(ctx *gin.Context) {
+ var req types.UserPromptsReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ds, total, err := h.c.Prompts(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get user prompts", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "message": "OK",
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+// GetUserEvaluations godoc
+// @Security ApiKey
+// @Summary Get user evaluations
+// @Description get user evaluations
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param per query int false "per" default(20)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/evaluations [get]
+func (h *UserHandler) GetEvaluations(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+ var req types.UserEvaluationReq
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ req.Owner = ctx.Param("username")
+ req.CurrentUser = httpbase.GetCurrentUser(ctx)
+ req.Page = page
+ req.PageSize = per
+ ds, total, err := h.c.Evaluations(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to get user evaluations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": ds,
+ "total": total,
+ }
+ ctx.JSON(http.StatusOK, respData)
+}
+
+
+
package httpbase
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+)
+
+// GracefulServer implements an HTTP server with graceful shutdown.
+// Graceful shutdown is actually hard to implement correctly
+// due to an API design flaw of the Go http package,
+// ref: https://nanmu.me/zh-cn/posts/2021/go-http-server-shudown-done-right/
+type GracefulServer struct {
+ server *http.Server
+}
+
+type GraceServerOpt struct {
+ Port int
+}
+
+// NewGracefulServer returns a server with graceful shutdown
+func NewGracefulServer(opt GraceServerOpt, handler http.Handler) (server *GracefulServer) {
+ server = &GracefulServer{
+ server: &http.Server{
+ Addr: fmt.Sprintf(":%d", opt.Port),
+ Handler: handler,
+ },
+ }
+ return
+}
+
+// Run start the http server and block
+func (s *GracefulServer) Run() {
+ q := make(chan os.Signal, 1)
+ signal.Notify(q, syscall.SIGINT, syscall.SIGTERM)
+
+ // Initializing the server in a goroutine so that
+ // it won't block the graceful shutdown handling below
+ go func() {
+ if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ //notify server to stop
+ q <- syscall.SIGTERM
+
+ slog.Error("listen failed", slog.Any("error", err))
+ }
+ }()
+
+ // Listen for the interrupt signal.
+ <-q
+
+ slog.Info("shutting down gracefully, press Ctrl+C again to force")
+
+ // The context is used to inform the server it has 5 seconds to finish
+ // the request it is currently handling
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := s.server.Shutdown(ctx); err != nil {
+ slog.Error("Server faild to shutdown", slog.Any("error", err))
+ }
+
+ slog.Info("Server stopped")
+}
+
+
+
package httpbase
+
+import (
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+)
+
+// OK responds the client with standard JSON.
+//
+// Example:
+// * ok(c, something)
+// * ok(c, nil)
+func OK(c *gin.Context, data interface{}) {
+ c.PureJSON(http.StatusOK, R{
+ Msg: "OK",
+ Data: data,
+ })
+}
+
+// BadRequest responds with a JSON-formatted error message.
+//
+// Example:
+//
+// BadRequest(c, "Invalid request parameters")
+func BadRequest(c *gin.Context, errMsg string) {
+ c.PureJSON(http.StatusBadRequest, R{
+ Msg: errMsg,
+ })
+}
+
+// ServerError responds with a JSON-formatted error message.
+//
+// Example:
+//
+// ServerError(c, errors.New("internal server error"))
+func ServerError(c *gin.Context, err error) {
+ c.PureJSON(http.StatusInternalServerError, R{
+ Msg: err.Error(),
+ })
+}
+
+// UnauthorizedError if the client is not authenticated or the authentication is invalid.
+// Like user not login, for example.
+//
+// Response Example:
+//
+// UnauthorizedError(c, errors.New("permission denied"))
+func UnauthorizedError(c *gin.Context, err error) {
+ c.PureJSON(http.StatusUnauthorized, R{
+ Msg: err.Error(),
+ })
+}
+
+// ForbiddenError if the client is authenticated but does not have enough permissions.
+func ForbiddenError(c *gin.Context, err error) {
+ c.PureJSON(http.StatusForbidden, R{
+ Msg: err.Error(),
+ })
+}
+
+// NotFoundError responds with a JSON-formatted error message.
+//
+// Example:
+//
+// NotFoundError(c, errors.New("permission denied"))
+func NotFoundError(c *gin.Context, err error) {
+ c.PureJSON(http.StatusNotFound, R{
+ Msg: err.Error(),
+ })
+}
+
+// R is the response envelope
+type R struct {
+ Code int `json:"code,omitempty"`
+ Msg string `json:"msg"`
+ Data any `json:"data,omitempty"`
+}
+
+
+
package httpbase
+
+import "github.com/gin-gonic/gin"
+
+const (
+ CurrentUserCtxVar = "currentUser"
+ CurrentUserUUIDCtxVar = "currentUserUUID"
+ AccessTokenCtxVar = "accessToken"
+ AuthTypeCtxVar = "authType"
+ CurrentUserQueryVar = "current_user"
+ CurrentUserUUIDQueryVar = "current_user_uuid"
+)
+
+type AuthType string
+
+const (
+ AuthTypeApiKey AuthType = "ApiKey"
+ AuthTypeJwt AuthType = "JWT"
+ AuthTypeAccessToken AuthType = "AccessToken"
+)
+
+// GetCurrentUser returns the current user name from the context.
+//
+// user name could be previously set by parsing query string or jwt token
+func GetCurrentUser(ctx *gin.Context) string {
+ return ctx.GetString(CurrentUserCtxVar)
+}
+
+func SetCurrentUser(ctx *gin.Context, user string) {
+ ctx.Set(CurrentUserCtxVar, user)
+}
+
+func GetAccessToken(ctx *gin.Context) string {
+ return ctx.GetString(AccessTokenCtxVar)
+}
+
+func SetAccessToken(ctx *gin.Context, user string) {
+ ctx.Set(AccessTokenCtxVar, user)
+}
+
+func GetAuthType(ctx *gin.Context) AuthType {
+ return AuthType(ctx.GetString(AuthTypeCtxVar))
+}
+
+func SetAuthType(ctx *gin.Context, t AuthType) {
+ ctx.Set(AuthTypeCtxVar, string(t))
+}
+
+func GetCurrentUserUUID(ctx *gin.Context) string {
+ return ctx.GetString(CurrentUserUUIDCtxVar)
+}
+
+func SetCurrentUserUUID(ctx *gin.Context, userUUID string) {
+ ctx.Set(CurrentUserUUIDCtxVar, userUUID)
+}
+
+
+
package middleware
+
+import (
+ "context"
+ "log/slog"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func GetUserFromAccessToken() gin.HandlerFunc {
+ userStore := database.NewUserStore()
+ return func(c *gin.Context) {
+ // Get Auzhorization token
+ authHeader := c.Request.Header.Get("Authorization")
+
+ if authHeader != "" {
+ // Get token
+ token := strings.TrimPrefix(authHeader, "Bearer ")
+ user, err := userStore.FindByAccessToken(context.Background(), token)
+ if err != nil {
+ slog.Debug("Can not find user by access token", slog.String("token", token))
+ c.Next()
+ return
+ }
+ if user != nil {
+ httpbase.SetCurrentUser(c, user.Username)
+ httpbase.SetAuthType(c, httpbase.AuthTypeAccessToken)
+ }
+ }
+
+ c.Next()
+ }
+}
+
+
+
package middleware
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "strings"
+
+ "github.com/gin-contrib/sessions"
+ "github.com/gin-gonic/gin"
+ "github.com/golang-jwt/jwt/v5"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+// BuildJwtSession create and save session with jwt from query string
+func BuildJwtSession(jwtSignKey string) gin.HandlerFunc {
+ return func(c *gin.Context) {
+ token := c.Query("jwt")
+
+ // If no JWT provided, continue with the next middleware
+ if token == "" {
+ c.Next()
+ return
+ }
+ claims, err := parseJWTToken(jwtSignKey, token)
+ if err != nil {
+ slog.Debug("fail to parse jwt token", slog.String("token_get", token), slog.Any("error", err))
+ c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+ return
+ }
+
+ sessions.Default(c).Set(httpbase.CurrentUserCtxVar, claims.CurrentUser)
+ err = sessions.Default(c).Save()
+ if err != nil {
+ slog.Error("fail to save session", slog.Any("error", err))
+ c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+ return
+ }
+
+ c.Next()
+ }
+}
+
+// AuthSession verify user login by session, ans save user name into context if login
+func AuthSession() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ session := sessions.Default(c)
+ userName := session.Get(httpbase.CurrentUserCtxVar)
+ if userName != nil {
+ httpbase.SetAuthType(c, httpbase.AuthTypeJwt)
+ httpbase.SetCurrentUser(c, userName.(string))
+ }
+
+ c.Next()
+ }
+}
+
+func Authenticator(config *config.Config) gin.HandlerFunc {
+ //TODO:change to component
+ userStore := database.NewUserStore()
+ return func(c *gin.Context) {
+ apiToken := config.APIToken
+
+ // Get Auzhorization token
+ authHeader := c.Request.Header.Get("Authorization")
+ if authHeader == "" {
+ c.Next()
+ return
+ }
+
+ if !strings.HasPrefix(authHeader, "Bearer ") {
+ httpbase.UnauthorizedError(c, errors.New("authorization header must starts with `Bearer `"))
+ c.Abort()
+ return
+ }
+ // Get token
+ token := strings.TrimPrefix(authHeader, "Bearer ")
+ if token == apiToken {
+ // get current user from query string
+ currentUser := c.Query(httpbase.CurrentUserQueryVar)
+ if len(currentUser) > 0 {
+ httpbase.SetCurrentUser(c, currentUser)
+ }
+ currentUserUUID := c.Query(httpbase.CurrentUserUUIDQueryVar)
+ if len(currentUserUUID) > 0 {
+ httpbase.SetCurrentUserUUID(c, currentUserUUID)
+ }
+ httpbase.SetAuthType(c, httpbase.AuthTypeApiKey)
+ c.Next()
+ return
+ }
+
+ if strings.Contains(token, ".") {
+ claims, err := parseJWTToken(config.JWT.SigningKey, token)
+ if err == nil {
+ httpbase.SetCurrentUser(c, claims.CurrentUser)
+ httpbase.SetCurrentUserUUID(c, claims.UUID)
+ httpbase.SetAuthType(c, httpbase.AuthTypeJwt)
+ return
+ }
+ } else {
+ //TODO:use cache to check access token
+ user, _ := userStore.FindByAccessToken(context.Background(), token)
+ if user != nil {
+ httpbase.SetCurrentUser(c, user.Username)
+ httpbase.SetCurrentUserUUID(c, user.UUID)
+ httpbase.SetAccessToken(c, token)
+ httpbase.SetAuthType(c, httpbase.AuthTypeAccessToken)
+ c.Next()
+ return
+ }
+ }
+
+ slog.ErrorContext(c, "invalid Bearer token", slog.String("token", token),
+ slog.String("ip", c.ClientIP()),
+ slog.String("method", c.Request.Method),
+ slog.String("url", c.Request.URL.RequestURI()),
+ )
+ httpbase.UnauthorizedError(c, errors.New("invalid Bearer token"))
+ c.Abort()
+ }
+}
+
+func parseJWTToken(signKey, tokenString string) (*types.JWTClaims, error) {
+ token, err := jwt.ParseWithClaims(tokenString, &types.JWTClaims{}, func(token *jwt.Token) (interface{}, error) {
+ return []byte(signKey), nil
+ })
+ if err != nil {
+ return nil, fmt.Errorf("invilid JWT token,%w", err)
+ }
+
+ if !token.Valid {
+ return nil, errors.New("invalid JWT token")
+ }
+
+ claims, ok := token.Claims.(*types.JWTClaims)
+ if ok {
+ return claims, nil
+ }
+ return nil, fmt.Errorf("JWT token claims not match: %+v", *token)
+}
+
+func OnlyAPIKeyAuthenticator(config *config.Config) gin.HandlerFunc {
+ return func(c *gin.Context) {
+ apiToken := config.APIToken
+
+ // Get Authorization token
+ authHeader := c.Request.Header.Get("Authorization")
+
+ // Check Authorization Header format
+ if authHeader == "" {
+ slog.Info("missing authorization header", slog.Any("url", c.Request.URL))
+ c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing Authorization header"})
+ return
+ }
+
+ // Get token
+ token := strings.TrimPrefix(authHeader, "Bearer ")
+
+ if token == apiToken {
+ // get current user from query string
+ currentUser := c.Query(httpbase.CurrentUserQueryVar)
+ if len(currentUser) > 0 {
+ httpbase.SetCurrentUser(c, currentUser)
+ }
+ } else {
+ c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "please use API key for authentication"})
+ return
+ }
+
+ c.Next()
+ }
+}
+
+func MustLogin() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("unknown user, please login first"))
+ ctx.Abort()
+ return
+ }
+ }
+}
+
+
+
package middleware
+
+import (
+ "compress/gzip"
+ "context"
+ "encoding/base64"
+ "fmt"
+ "io"
+ "log/slog"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+const gitSuffix = ".git"
+
+func GitHTTPParamMiddleware() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ name := c.Param("name")
+ namespace := c.Param("namespace")
+ repo_type := strings.TrimSuffix(c.Param("repo_type"), "s")
+
+ if namespace == "" {
+ httpbase.BadRequest(c, "invalid repository namespace")
+ return
+ }
+
+ if repo_type == "" {
+ httpbase.BadRequest(c, "invalid repository repository type")
+ return
+ }
+
+ if name == "" || strings.TrimSuffix(name, gitSuffix) == "" {
+ httpbase.BadRequest(c, "invalid repository name")
+ return
+ }
+
+ if strings.HasSuffix(name, gitSuffix) {
+ c.Set("name", strings.TrimSuffix(name, gitSuffix))
+ } else {
+ c.Set("name", name)
+ }
+ c.Set("namespace", namespace)
+ c.Set("repo_type", repo_type)
+
+ c.Next()
+ }
+}
+
+func ContentEncoding() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ var (
+ body io.ReadCloser
+ err error
+ )
+
+ contentEncoding := c.Request.Header.Get("Content-Encoding")
+ switch contentEncoding {
+ case "":
+ body = c.Request.Body
+ case "gzip":
+ body, err = gzip.NewReader(c.Request.Body)
+ default:
+ err = fmt.Errorf("unsupported content encoding: %s", contentEncoding)
+ }
+
+ if err != nil {
+ httpbase.BadRequest(c, err.Error())
+ c.Abort()
+ return
+ }
+ defer body.Close()
+
+ c.Request.Body = body
+ c.Request.Header.Del("Content-Encoding")
+
+ c.Next()
+ }
+}
+
+func GetCurrentUserFromHeader() gin.HandlerFunc {
+ userStore := database.NewUserStore()
+ return func(c *gin.Context) {
+ authHeader := c.Request.Header.Get("Authorization")
+ if authHeader != "" && !strings.HasPrefix(authHeader, "X-OPENCSG-Sync-Token") {
+ authHeader = strings.TrimPrefix(authHeader, "Basic ")
+ authInfo, err := base64.StdEncoding.DecodeString(authHeader)
+ if err != nil {
+ slog.Info("Failed to decode basic auth header", slog.Any("header", authHeader), slog.Any("error", err))
+ c.Next()
+ return
+ }
+ username := strings.Split(string(authInfo), ":")[0]
+ password := strings.Split(string(authInfo), ":")[1]
+
+ user, err := userStore.FindByGitAccessToken(context.Background(), password)
+ if err != nil {
+ slog.Info("Failed to find user by git access token", slog.Any("header", authHeader), slog.Any("token", password), slog.Any("error", err))
+ c.Next()
+ return
+ }
+ if user.Username == username {
+ httpbase.SetCurrentUser(c, username)
+ }
+ }
+
+ c.Next()
+ }
+}
+
+
+
package middleware
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "github.com/golang-jwt/jwt/v5"
+ "opencsg.com/csghub-server/common/config"
+)
+
+const apiSecretHeaderName = "Gitlab-Shell-Api-Request"
+
+func parseGitlabShellJWTToken(signKey, tokenString string) (bool, error) {
+ token, err := jwt.ParseWithClaims(tokenString, &jwt.RegisteredClaims{}, func(token *jwt.Token) (interface{}, error) {
+ return []byte(signKey), nil
+ })
+ if err != nil {
+ return false, fmt.Errorf("invilid JWT token,%w", err)
+ }
+
+ if !token.Valid {
+ return false, errors.New("invalid JWT token")
+ }
+ return true, nil
+}
+
+func CheckGitlabShellJWTToken(config *config.Config) gin.HandlerFunc {
+ return func(c *gin.Context) {
+ tokenString := c.Request.Header.Get(apiSecretHeaderName)
+ pass, err := parseGitlabShellJWTToken(config.GitalyServer.JWTSecret, tokenString)
+ if err != nil {
+ slog.Debug("fail to parse gitlab-shell jwt token", slog.String("token_get", tokenString), slog.Any("error", err))
+ c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+ return
+ }
+ if !pass {
+ slog.Debug("invalid gilab-shell jwt token", slog.String("token_get", tokenString))
+ c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid gitlab-shell jwt token"})
+ return
+ }
+ c.Next()
+ }
+}
+
+
+
package middleware
+
+import (
+ "context"
+ "log/slog"
+ "os"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+)
+
+func Log() gin.HandlerFunc {
+ lh := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+ AddSource: false,
+ Level: slog.LevelInfo,
+ })
+ l := slog.New(lh)
+ return func(ctx *gin.Context) {
+ startTime := time.Now()
+ ctx.Set("clientIP", ctx.ClientIP())
+ reqCtx := context.WithValue(ctx.Request.Context(), "clientIP", ctx.ClientIP())
+ ctx.Request = ctx.Request.WithContext(reqCtx)
+
+ ctx.Next()
+
+ latency := time.Since(startTime).Milliseconds()
+ l.InfoContext(ctx, "http request", slog.String("ip", ctx.ClientIP()),
+ slog.String("method", ctx.Request.Method),
+ slog.Int("latency(ms)", int(latency)),
+ slog.Int("status", ctx.Writer.Status()),
+ slog.String("current_user", httpbase.GetCurrentUser(ctx)),
+ slog.Any("auth_type", httpbase.GetAuthType(ctx)),
+ slog.String("url", ctx.Request.URL.RequestURI()),
+ )
+ }
+}
+
+
+
package middleware
+
+import (
+ "log/slog"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func RepoType(t types.RepositoryType) gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ slog.Debug("middleware RepoType called", "repo_type", t)
+ common.SetRepoTypeContext(ctx, t)
+ ctx.Next()
+ }
+}
+
+func RepoMapping(repo_type types.RepositoryType) gin.HandlerFunc {
+ mirrorStore := database.NewMirrorStore()
+ return func(ctx *gin.Context) {
+ slog.Debug("middleware RepoMapping called")
+ common.SetRepoTypeContext(ctx, repo_type)
+ namespace := ctx.Param("namespace")
+ name := ctx.Param("name")
+ branch := ctx.Param("branch")
+ if branch == "" {
+ branch = ctx.Param("ref")
+ }
+ mapping := GetMapping(ctx)
+ if mapping == types.CSGHubMapping {
+ ctx.Next()
+ return
+ }
+ mirror, err := mirrorStore.FindWithMapping(ctx, repo_type, namespace, name, mapping)
+ //if found mirror, that means this is a synced source, otherwise it's may a user-upload repo
+ if err == nil {
+ repo_id := strings.Split(mirror.Repository.Path, "/")
+ //set the real namespace, the name was unchange
+ slog.Info("namespace changed: ", "namespace", repo_id[0])
+ ctx.Set("namespace_mapped", repo_id[0])
+ ctx.Set("name_mapped", repo_id[1])
+ // for modelscope, the default branch is master, we should map it to real branch
+ if (branch == "main" || branch == "master") && mirror.Repository.DefaultBranch != branch {
+ ctx.Set("branch_mapped", mirror.Repository.DefaultBranch)
+ }
+ ctx.Next()
+ return
+ }
+ ctx.Next()
+ }
+}
+
+func GetMapping(ctx *gin.Context) types.Mapping {
+ rawRp := ctx.Query("mirror")
+ if rawRp == "" {
+ return types.AutoMapping
+ }
+ return types.Mapping(rawRp)
+}
+
+
+
package router
+
+import (
+ "fmt"
+ "time"
+
+ cache "github.com/chenyahui/gin-cache"
+ "github.com/chenyahui/gin-cache/persist"
+ "github.com/gin-contrib/cors"
+ "github.com/gin-gonic/gin"
+ swaggerFiles "github.com/swaggo/files"
+ ginSwagger "github.com/swaggo/gin-swagger"
+ "opencsg.com/csghub-server/api/handler"
+ "opencsg.com/csghub-server/api/handler/callback"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func NewRouter(config *config.Config, enableSwagger bool) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(cors.New(cors.Config{
+ AllowCredentials: true,
+ AllowHeaders: []string{"*"},
+ AllowMethods: []string{"*"},
+ AllowAllOrigins: true,
+ }))
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+
+ gitHTTPHandler, err := handler.NewGitHTTPHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating git http handler:%w", err)
+ }
+ gitHTTP := r.Group("/:repo_type/:namespace/:name")
+ gitHTTP.Use(middleware.GitHTTPParamMiddleware())
+ gitHTTP.Use(middleware.GetCurrentUserFromHeader())
+ {
+ gitHTTP.GET("/info/refs", gitHTTPHandler.InfoRefs)
+ gitHTTP.POST("/git-upload-pack", middleware.ContentEncoding(), gitHTTPHandler.GitUploadPack)
+ gitHTTP.POST("/git-receive-pack", middleware.ContentEncoding(), gitHTTPHandler.GitReceivePack)
+ // gitHTTP.GET("/objects/info/alternates", gitHTTPHandler.Serve)
+ // gitHTTP.GET("/objects/info/http-alternates", gitHTTPHandler.Serve)
+ // gitHTTP.GET("/objects/info/packs", gitHTTPHandler.Serve)
+ // gitHTTP.GET("/objects/info/:file", gitHTTPHandler.Serve)
+ // gitHTTP.GET("/objects/:head/:hash", gitHTTPHandler.Serve)
+ // gitHTTP.GET("/objects/pack/pack-:file", gitHTTPHandler.Serve)
+ lfsGroup := gitHTTP.Group("/info/lfs")
+ {
+ objectsGroup := lfsGroup.Group("/objects")
+ {
+ objectsGroup.POST("/batch", gitHTTPHandler.LfsBatch)
+ objectsGroup.PUT("/:oid/:size", gitHTTPHandler.LfsUpload)
+ lfsGroup.GET("/:oid", gitHTTPHandler.LfsDownload)
+ }
+ lfsGroup.POST("/verify", gitHTTPHandler.LfsVerify)
+
+ locksGroup := lfsGroup.Group("/locks")
+ {
+ locksGroup.GET("", gitHTTPHandler.ListLocks)
+ locksGroup.POST("", gitHTTPHandler.CreateLock)
+ locksGroup.POST("/verify", gitHTTPHandler.VerifyLock)
+ locksGroup.POST("/:lid/unlock", gitHTTPHandler.UnLock)
+ }
+
+ }
+
+ }
+
+ engine, err2 := createPaymentRoutes(config, r)
+ if err2 != nil {
+ return engine, err2
+ }
+
+ r.Use(middleware.Authenticator(config))
+
+ needAPIKey := middleware.OnlyAPIKeyAuthenticator(config)
+
+ if enableSwagger {
+ r.GET("/api/v1/swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))
+ }
+
+ // User routes
+ userHandler, err := handler.NewUserHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating user controller:%w", err)
+ }
+ orgHandler, err := handler.NewOrganizationHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating organization controller:%w", err)
+ }
+
+ repoCommonHandler, err := handler.NewRepoHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating repo common handler: %w", err)
+ }
+ modelHandler, err := handler.NewModelHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating model controller:%w", err)
+ }
+ dsHandler, err := handler.NewDatasetHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating dataset handler:%w", err)
+ }
+
+ // Mirror
+ mirrorHandler, err := handler.NewMirrorHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating mirror controller:%w", err)
+ }
+
+ hfdsHandler, err := handler.NewHFDatasetHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating HF dataset handler: %w", err)
+ }
+
+ createHFRoutes(r, hfdsHandler, repoCommonHandler, modelHandler, userHandler)
+
+ apiGroup := r.Group("/api/v1")
+ // TODO:use middleware to handle common response
+ //
+ memoryStore := persist.NewMemoryStore(1 * time.Minute)
+
+ // List trending models and datasets routes
+ listHandler, err := handler.NewListHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creatring list handler: %v", err)
+ }
+ {
+ apiGroup.POST("/list/models_by_path", cache.CacheByRequestURI(memoryStore, 1*time.Minute), listHandler.ListModelsByPath)
+ apiGroup.POST("/list/datasets_by_path", cache.CacheByRequestURI(memoryStore, 1*time.Minute), listHandler.ListDatasetsByPath)
+ apiGroup.POST("/list/spaces_by_path", cache.CacheByRequestURI(memoryStore, 1*time.Minute), listHandler.ListSpacesByPath)
+ }
+
+ //evaluation handler
+ evaluationHandler, err := handler.NewEvaluationHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creatring evaluation handler: %v", err)
+ }
+
+ createEvaluationRoutes(apiGroup, evaluationHandler)
+
+ // Model routes
+ createModelRoutes(config, apiGroup, needAPIKey, modelHandler, repoCommonHandler)
+
+ // Dataset viewer
+ dsViewerHandler, err := handler.NewDatasetViewerHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating dataset viewer handler:%w", err)
+ }
+
+ // Dataset routes
+ createDatasetRoutes(config, apiGroup, dsHandler, repoCommonHandler, dsViewerHandler)
+
+ codeHandler, err := handler.NewCodeHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating code handler:%w", err)
+ }
+ // Code routes
+ createCodeRoutes(config, apiGroup, codeHandler, repoCommonHandler)
+
+ spaceHandler, err := handler.NewSpaceHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating space handler:%w", err)
+ }
+ // space routers
+ createSpaceRoutes(config, apiGroup, spaceHandler, repoCommonHandler)
+
+ spaceResourceHandler, err := handler.NewSpaceResourceHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating space resource handler:%w", err)
+ }
+
+ spaceResource := apiGroup.Group("space_resources")
+ {
+ spaceResource.GET("", spaceResourceHandler.Index)
+ spaceResource.POST("", needAPIKey, spaceResourceHandler.Create)
+ spaceResource.PUT("/:id", needAPIKey, spaceResourceHandler.Update)
+ spaceResource.DELETE("/:id", needAPIKey, spaceResourceHandler.Delete)
+ }
+
+ spaceSdkHandler, err := handler.NewSpaceSdkHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating space sdk handler:%w", err)
+ }
+
+ spaceSdk := apiGroup.Group("space_sdks")
+ {
+ spaceSdk.GET("", spaceSdkHandler.Index)
+ spaceSdk.POST("", needAPIKey, spaceSdkHandler.Create)
+ spaceSdk.PUT("/:id", needAPIKey, spaceSdkHandler.Update)
+ spaceSdk.DELETE("/:id", needAPIKey, spaceSdkHandler.Delete)
+ }
+
+ userProxyHandler, err := handler.NewInternalServiceProxyHandler(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port))
+ if err != nil {
+ return nil, fmt.Errorf("error creating user proxy handler:%w", err)
+ }
+
+ createUserRoutes(apiGroup, needAPIKey, userProxyHandler, userHandler)
+
+ tokenGroup := apiGroup.Group("token")
+ {
+ tokenGroup.POST("/:app/:token_name", userProxyHandler.ProxyToApi("/api/v1/token/%s/%s", "app", "token_name"))
+ tokenGroup.PUT("/:app/:token_name", userProxyHandler.ProxyToApi("/api/v1/token/%s/%s", "app", "token_name"))
+ tokenGroup.DELETE("/:app/:token_name", userProxyHandler.ProxyToApi("/api/v1/token/%s/%s", "app", "token_name"))
+ // check token info
+ tokenGroup.GET("/:token_value", needAPIKey, userProxyHandler.ProxyToApi("/api/v1/token/%s", "token_value"))
+ }
+
+ sshKeyHandler, err := handler.NewSSHKeyHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating user controller:%w", err)
+ }
+ {
+ apiGroup.GET("/user/:username/ssh_keys", sshKeyHandler.Index)
+ apiGroup.POST("/user/:username/ssh_keys", sshKeyHandler.Create)
+ apiGroup.DELETE("/user/:username/ssh_key/:name", sshKeyHandler.Delete)
+ }
+
+ {
+ apiGroup.GET("/organizations", userProxyHandler.Proxy)
+ apiGroup.POST("/organizations", userProxyHandler.Proxy)
+ apiGroup.GET("/organization/:namespace", userProxyHandler.ProxyToApi("/api/v1/organization/%s", "namespace"))
+ apiGroup.PUT("/organization/:namespace", userProxyHandler.ProxyToApi("/api/v1/organization/%s", "namespace"))
+ apiGroup.DELETE("/organization/:namespace", userProxyHandler.ProxyToApi("/api/v1/organization/%s", "namespace"))
+ // Organization assets
+ apiGroup.GET("/organization/:namespace/models", orgHandler.Models)
+ apiGroup.GET("/organization/:namespace/datasets", orgHandler.Datasets)
+ apiGroup.GET("/organization/:namespace/codes", orgHandler.Codes)
+ apiGroup.GET("/organization/:namespace/spaces", orgHandler.Spaces)
+ apiGroup.GET("/organization/:namespace/collections", orgHandler.Collections)
+ apiGroup.GET("/organization/:namespace/prompts", orgHandler.Prompts)
+ }
+
+ {
+ apiGroup.GET("/organization/:namespace/members", userProxyHandler.ProxyToApi("/api/v1/organization/%s/members", "namespace"))
+ apiGroup.POST("/organization/:namespace/members", userProxyHandler.ProxyToApi("/api/v1/organization/%s/members", "namespace"))
+ apiGroup.GET("/organization/:namespace/members/:username", userProxyHandler.ProxyToApi("/api/v1/organization/%s/members/%s", "namespace", "username"))
+ apiGroup.PUT("/organization/:namespace/members/:username", userProxyHandler.ProxyToApi("/api/v1/organization/%s/members/%s", "namespace", "username"))
+ apiGroup.DELETE("/organization/:namespace/members/:username", userProxyHandler.ProxyToApi("/api/v1/organization/%s/members/%s", "namespace", "username"))
+ }
+ // Tag
+ tagCtrl, err := handler.NewTagHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating tag controller:%w", err)
+ }
+ apiGroup.GET("/tags", tagCtrl.AllTags)
+ // apiGroup.POST("/tag", tagCtrl.NewTag)
+ // apiGroup.PUT("/tag", tagCtrl.UpdateTag)
+ // apiGroup.DELETE("/tag", tagCtrl.DeleteTag)
+
+ // JWT token
+ apiGroup.POST("/jwt/token", needAPIKey, userProxyHandler.Proxy)
+ apiGroup.GET("/jwt/:token", needAPIKey, userProxyHandler.ProxyToApi("/api/v1/jwt/%s", "token"))
+ apiGroup.GET("/users", userProxyHandler.Proxy)
+
+ // callback
+ callbackCtrl, err := callback.NewGitCallbackHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating callback controller:%w", err)
+ }
+ apiGroup.POST("/callback/git", callbackCtrl.Handle)
+ apiGroup.GET("/callback/casdoor", userProxyHandler.Proxy)
+ // Sensive check
+ if config.SensitiveCheck.Enable {
+ sensitiveCtrl, err := handler.NewSensitiveHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive handler:%w", err)
+ }
+ apiGroup.POST("/sensitive/text", sensitiveCtrl.Text)
+ apiGroup.POST("/sensitive/image", sensitiveCtrl.Image)
+ }
+
+ // MirrorSource
+ msHandler, err := handler.NewMirrorSourceHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating mirror source controller:%w", err)
+ }
+
+ apiGroup.GET("/mirrors", mirrorHandler.Index)
+
+ mirror := apiGroup.Group("/mirror")
+ {
+ mirror.GET("/sources", msHandler.Index)
+ mirror.POST("/sources", msHandler.Create)
+ mirror.PUT("/sources/:id", msHandler.Update)
+ mirror.DELETE("/sources/:id", msHandler.Delete)
+ mirror.GET("/sources/:id", msHandler.Get)
+ mirror.POST("/repo", mirrorHandler.CreateMirrorRepo)
+ mirror.GET("/repos", mirrorHandler.Repos)
+ mirror.GET("/statistics", mirrorHandler.Statistics)
+
+ }
+
+ collectionHandler, err := handler.NewCollectionHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating collection handler:%w", err)
+ }
+ collections := apiGroup.Group("/collections")
+ {
+ // list all collection
+ collections.GET("", collectionHandler.Index)
+ collections.POST("", collectionHandler.Create)
+ collections.GET("/:id", collectionHandler.GetCollection)
+ collections.PUT("/:id", collectionHandler.UpdateCollection)
+ collections.DELETE("/:id", collectionHandler.DeleteCollection)
+ collections.POST("/:id/repos", collectionHandler.AddRepoToCollection)
+ collections.DELETE("/:id/repos", collectionHandler.RemoveRepoFromCollection)
+ }
+
+ // cluster infos
+ clusterHandler, err := handler.NewClusterHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to creating cluster handler: %w", err)
+ }
+ cluster := apiGroup.Group("/cluster")
+ {
+ cluster.GET("", clusterHandler.Index)
+ cluster.GET("/:id", clusterHandler.GetClusterById)
+ cluster.PUT("/:id", needAPIKey, clusterHandler.Update)
+ }
+
+ eventHandler, err := handler.NewEventHandler()
+ if err != nil {
+ return nil, fmt.Errorf("error creating event handler:%w", err)
+ }
+ event := apiGroup.Group("/events")
+ event.POST("", eventHandler.Create)
+
+ runtimeArchHandler, err := handler.NewRuntimeArchitectureHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating runtime framework architecture handler:%w", err)
+ }
+
+ createRuntimeFrameworkRoutes(apiGroup, needAPIKey, modelHandler, runtimeArchHandler)
+
+ syncHandler, err := handler.NewSyncHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sync handler:%w", err)
+ }
+ syncClientSettingHandler, err := handler.NewSyncClientSettingHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sync client setting handler:%w", err)
+ }
+ syncGroup := apiGroup.Group("sync")
+ {
+ syncGroup.GET("/version/latest", syncHandler.Latest)
+ // syncGroup.GET("/version/oldest", syncHandler.Oldest)
+ syncGroup.GET("/client_setting", syncClientSettingHandler.Show)
+ syncGroup.POST("/client_setting", syncClientSettingHandler.Create)
+ }
+
+ accountingHandler, err := handler.NewAccountingHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating accounting handler setting handler:%w", err)
+ }
+
+ accountingProxyHandler, err := handler.NewInternalServiceProxyHandler(fmt.Sprintf("%s:%d", config.Accounting.Host, config.Accounting.Port))
+ if err != nil {
+ return nil, fmt.Errorf("error creating accountingProxyHandler:%w", err)
+ }
+ createAccountRoutes(apiGroup, needAPIKey, accountingHandler, accountingProxyHandler)
+
+ recomHandler, err := handler.NewRecomHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating recomHandler,%w", err)
+ }
+ recomGroup := apiGroup.Group("/recom")
+ {
+ recomGroup.POST("opweight", recomHandler.SetOpWeight)
+ }
+
+ // telemetry
+ telemetryHandler, err := handler.NewTelemetryHandler()
+ if err != nil {
+ return nil, fmt.Errorf("error creating telemetry handler:%w", err)
+ }
+ teleGroup := apiGroup.Group("/telemetry")
+ teleGroup.POST("/usage", telemetryHandler.Usage)
+
+ // internal API for gitaly to check request permissions
+ internalHandler, err := handler.NewInternalHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating internalHandler,%w", err)
+ }
+ needGitlabShellJWTToken := middleware.CheckGitlabShellJWTToken(config)
+ r.GET("/api/v4/internal/authorized_keys", needGitlabShellJWTToken, internalHandler.GetAuthorizedKeys)
+ r.POST("/api/v4/internal/allowed", needGitlabShellJWTToken, internalHandler.SSHAllowed)
+ r.POST("/api/v4/internal/pre_receive", needGitlabShellJWTToken, internalHandler.PreReceive)
+ r.POST("api/v4/internal/lfs_authenticate", needGitlabShellJWTToken, internalHandler.LfsAuthenticate)
+ r.POST("/api/v4/internal/post_receive", needGitlabShellJWTToken, internalHandler.PostReceive)
+ internalGroup := apiGroup.Group("/internal")
+ {
+ internalGroup.POST("/allowed", needGitlabShellJWTToken, internalHandler.Allowed)
+ internalGroup.POST("/pre_receive", needGitlabShellJWTToken, internalHandler.PreReceive)
+ internalGroup.POST("/post_receive", needGitlabShellJWTToken, internalHandler.PostReceive)
+ }
+
+ discussionHandler, err := handler.NewDiscussionHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating discussion handler:%w", err)
+ }
+ createDiscussionRoutes(apiGroup, discussionHandler)
+
+ // prompt
+ promptHandler, err := handler.NewPromptHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating prompt handler,%w", err)
+ }
+ createPromptRoutes(apiGroup, promptHandler)
+
+ // dataflow proxy
+ dataflowHandler, err := handler.NewDataflowProxyHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating data flow proxy handler:%w", err)
+ }
+ createDataflowRoutes(apiGroup, dataflowHandler)
+
+ // license
+ licenseHandler, err := handler.NewLicenseHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating license handler:%w", err)
+ }
+ createLicenseRoutes(apiGroup, licenseHandler)
+
+ return r, nil
+}
+
+func createPaymentRoutes(config *config.Config, r *gin.Engine) (*gin.Engine, error) {
+ paymentProxyHandler, err := handler.NewInternalServiceProxyHandler(fmt.Sprintf("%s:%d", config.Payment.Host, config.Payment.Port))
+ if err != nil {
+ return nil, fmt.Errorf("error creating payment controller: %w", err)
+ }
+ r.POST("/api/v1/payment/wechat/notify", paymentProxyHandler.Proxy)
+ r.POST("/api/v1/payment/alipay/notify", paymentProxyHandler.Proxy)
+ return r, nil
+}
+
+func createEvaluationRoutes(apiGroup *gin.RouterGroup, evaluationHandler *handler.EvaluationHandler) {
+ // Models routes
+ evaluationsGroup := apiGroup.Group("/evaluations")
+ {
+ evaluationsGroup.POST("", evaluationHandler.RunEvaluation)
+ evaluationsGroup.DELETE("/:id", evaluationHandler.DeleteEvaluation)
+ evaluationsGroup.GET("/:id", evaluationHandler.GetEvaluation)
+ }
+}
+
+func createModelRoutes(config *config.Config, apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, modelHandler *handler.ModelHandler, repoCommonHandler *handler.RepoHandler) {
+ // Models routes
+ modelsGroup := apiGroup.Group("/models")
+ {
+ modelsGroup.POST("", modelHandler.Create)
+ modelsGroup.GET("", modelHandler.Index)
+ modelsGroup.PUT("/:namespace/:name", modelHandler.Update)
+ modelsGroup.DELETE("/:namespace/:name", modelHandler.Delete)
+ modelsGroup.GET("/:namespace/:name", modelHandler.Show)
+ modelsGroup.GET("/:namespace/:name/all_files", modelHandler.AllFiles)
+ modelsGroup.GET("/:namespace/:name/relations", modelHandler.Relations)
+ modelsGroup.PUT("/:namespace/:name/relations", modelHandler.SetRelations)
+ modelsGroup.POST("/:namespace/:name/relations/dataset", modelHandler.AddDatasetRelation)
+ modelsGroup.DELETE("/:namespace/:name/relations/dataset", modelHandler.DelDatasetRelation)
+ modelsGroup.GET("/:namespace/:name/branches", middleware.RepoType(types.ModelRepo), repoCommonHandler.Branches)
+ modelsGroup.GET("/:namespace/:name/tags", middleware.RepoType(types.ModelRepo), repoCommonHandler.Tags)
+ // update tags of a certain category
+ modelsGroup.POST("/:namespace/:name/tags/:category", middleware.RepoType(types.ModelRepo), repoCommonHandler.UpdateTags)
+ modelsGroup.GET("/:namespace/:name/last_commit", middleware.RepoType(types.ModelRepo), repoCommonHandler.LastCommit)
+ modelsGroup.GET("/:namespace/:name/commit/:commit_id", middleware.RepoType(types.ModelRepo), repoCommonHandler.CommitWithDiff)
+ modelsGroup.GET("/:namespace/:name/tree", middleware.RepoType(types.ModelRepo), repoCommonHandler.Tree)
+ modelsGroup.GET("/:namespace/:name/commits", middleware.RepoType(types.ModelRepo), repoCommonHandler.Commits)
+ modelsGroup.GET("/:namespace/:name/raw/*file_path", middleware.RepoType(types.ModelRepo), repoCommonHandler.FileRaw)
+ modelsGroup.GET("/:namespace/:name/blob/*file_path", middleware.RepoType(types.ModelRepo), repoCommonHandler.FileInfo)
+ // The DownloadFile method differs from the SDKDownload interface in a few ways
+
+ // 1.When passing the file_path parameter to the SDKDownload method,
+ // it only needs to pass the path of the file itself,
+ // whether it is an lfs file or a non-lfs file.
+ // The DownloadFile has a different file_path format for lfs files and non-lfs files,
+ // and an lfs parameter needs to be added.
+ // 2. DownloadFile returns an object store url for lfs files, while SDKDownload redirects directly.
+ modelsGroup.GET("/:namespace/:name/download/*file_path", middleware.RepoType(types.ModelRepo), repoCommonHandler.DownloadFile)
+ modelsGroup.GET("/:namespace/:name/resolve/*file_path", middleware.RepoType(types.ModelRepo), repoCommonHandler.ResolveDownload)
+ modelsGroup.POST("/:namespace/:name/raw/*file_path", middleware.RepoType(types.ModelRepo), repoCommonHandler.CreateFile)
+ modelsGroup.PUT("/:namespace/:name/raw/*file_path", middleware.RepoType(types.ModelRepo), repoCommonHandler.UpdateFile)
+ modelsGroup.POST("/:namespace/:name/update_downloads", middleware.RepoType(types.ModelRepo), repoCommonHandler.UpdateDownloads)
+ modelsGroup.PUT("/:namespace/:name/incr_downloads", middleware.RepoType(types.ModelRepo), repoCommonHandler.IncrDownloads)
+ modelsGroup.POST("/:namespace/:name/upload_file", middleware.RepoType(types.ModelRepo), repoCommonHandler.UploadFile)
+ modelsGroup.POST("/:namespace/:name/mirror", middleware.RepoType(types.ModelRepo), repoCommonHandler.CreateMirror)
+ modelsGroup.GET("/:namespace/:name/mirror", middleware.RepoType(types.ModelRepo), repoCommonHandler.GetMirror)
+ modelsGroup.PUT("/:namespace/:name/mirror", middleware.RepoType(types.ModelRepo), repoCommonHandler.UpdateMirror)
+ modelsGroup.DELETE("/:namespace/:name/mirror", middleware.RepoType(types.ModelRepo), repoCommonHandler.DeleteMirror)
+ modelsGroup.POST("/:namespace/:name/mirror/sync", middleware.RepoType(types.ModelRepo), repoCommonHandler.SyncMirror)
+ modelsGroup.GET("/:namespace/:name/mirror/progress", middleware.RepoType(types.ModelRepo), repoCommonHandler.MirrorProgress)
+
+ // mirror from SaaS, only on-premises available
+ if !config.Saas {
+ modelsGroup.POST("/:namespace/:name/mirror_from_saas", middleware.RepoType(types.ModelRepo), repoCommonHandler.MirrorFromSaas)
+ }
+
+ // runtime framework
+ modelsGroup.GET("/:namespace/:name/runtime_framework", middleware.RepoType(types.ModelRepo), repoCommonHandler.RuntimeFrameworkList)
+ modelsGroup.POST("/:namespace/:name/runtime_framework", needAPIKey, middleware.RepoType(types.ModelRepo), repoCommonHandler.RuntimeFrameworkCreate)
+ modelsGroup.PUT("/:namespace/:name/runtime_framework/:id", needAPIKey, middleware.RepoType(types.ModelRepo), repoCommonHandler.RuntimeFrameworkUpdate)
+ modelsGroup.DELETE("/:namespace/:name/runtime_framework/:id", needAPIKey, middleware.RepoType(types.ModelRepo), repoCommonHandler.RuntimeFrameworkDelete)
+ // list model inference
+ modelsGroup.GET("/:namespace/:name/run", middleware.RepoType(types.ModelRepo), repoCommonHandler.DeployList)
+ // deploy model as inference
+ modelsGroup.POST("/:namespace/:name/run", middleware.RepoType(types.ModelRepo), modelHandler.DeployDedicated)
+ // delete a deployed inference
+ modelsGroup.DELETE("/:namespace/:name/run/:id", middleware.RepoType(types.ModelRepo), modelHandler.DeployDelete)
+ modelsGroup.GET("/:namespace/:name/run/:id", middleware.RepoType(types.ModelRepo), repoCommonHandler.DeployDetail)
+ modelsGroup.GET("/:namespace/:name/run/:id/status", middleware.RepoType(types.ModelRepo), repoCommonHandler.DeployStatus)
+ modelsGroup.GET("/:namespace/:name/run/:id/logs/:instance", middleware.RepoType(types.ModelRepo), repoCommonHandler.DeployInstanceLogs)
+ modelsGroup.PUT("/:namespace/:name/run/:id", middleware.RepoType(types.ModelRepo), repoCommonHandler.DeployUpdate)
+ modelsGroup.PUT("/:namespace/:name/run/:id/stop", middleware.RepoType(types.ModelRepo), modelHandler.DeployStop)
+ modelsGroup.PUT("/:namespace/:name/run/:id/start", middleware.RepoType(types.ModelRepo), modelHandler.DeployStart)
+
+ // runtime framework for both finetune and inference
+ modelsGroup.GET("/runtime_framework", middleware.RepoType(types.ModelRepo), repoCommonHandler.RuntimeFrameworkListWithType)
+
+ // deploy model as finetune instance
+ modelsGroup.POST("/:namespace/:name/finetune", middleware.RepoType(types.ModelRepo), modelHandler.FinetuneCreate)
+ // stop a finetune instance
+ modelsGroup.PUT("/:namespace/:name/finetune/:id/stop", middleware.RepoType(types.ModelRepo), modelHandler.FinetuneStop)
+ // start a finetune instance
+ modelsGroup.PUT("/:namespace/:name/finetune/:id/start", middleware.RepoType(types.ModelRepo), modelHandler.FinetuneStart)
+ // delete a finetune instance
+ modelsGroup.DELETE("/:namespace/:name/finetune/:id", middleware.RepoType(types.ModelRepo), modelHandler.FinetuneDelete)
+
+ // deploy model as serverless
+ modelsGroup.GET("/:namespace/:name/serverless", middleware.RepoType(types.ModelRepo), modelHandler.GetDeployServerless)
+ modelsGroup.POST("/:namespace/:name/serverless", middleware.RepoType(types.ModelRepo), modelHandler.DeployServerless)
+ modelsGroup.PUT("/:namespace/:name/serverless/:id/start", middleware.RepoType(types.ModelRepo), modelHandler.ServerlessStart)
+ modelsGroup.PUT("/:namespace/:name/serverless/:id/stop", middleware.RepoType(types.ModelRepo), modelHandler.ServerlessStop)
+ modelsGroup.GET("/:namespace/:name/serverless/:id", middleware.RepoType(types.ModelRepo), repoCommonHandler.ServerlessDetail)
+ modelsGroup.GET("/:namespace/:name/serverless/:id/status", middleware.RepoType(types.ModelRepo), repoCommonHandler.ServerlessStatus)
+ modelsGroup.GET("/:namespace/:name/serverless/:id/logs/:instance", middleware.RepoType(types.ModelRepo), repoCommonHandler.ServerlessLogs)
+ modelsGroup.PUT("/:namespace/:name/serverless/:id", middleware.RepoType(types.ModelRepo), repoCommonHandler.ServerlessUpdate)
+ }
+}
+
+func createDatasetRoutes(config *config.Config, apiGroup *gin.RouterGroup, dsHandler *handler.DatasetHandler, repoCommonHandler *handler.RepoHandler, dsViewerHandler *handler.DatasetViewerHandler) {
+ datasetsGroup := apiGroup.Group("/datasets")
+ // allow access without login
+ datasetsGroup.GET("", dsHandler.Index)
+ // must login
+ datasetsGroup.Use(middleware.MustLogin())
+ {
+ datasetsGroup.POST("", dsHandler.Create)
+ datasetsGroup.PUT("/:namespace/:name", dsHandler.Update)
+ datasetsGroup.DELETE("/:namespace/:name", dsHandler.Delete)
+ datasetsGroup.GET("/:namespace/:name", dsHandler.Show)
+ datasetsGroup.GET("/:namespace/:name/all_files", dsHandler.AllFiles)
+ datasetsGroup.GET("/:namespace/:name/relations", dsHandler.Relations)
+ datasetsGroup.GET("/:namespace/:name/branches", middleware.RepoType(types.DatasetRepo), repoCommonHandler.Branches)
+ datasetsGroup.GET("/:namespace/:name/tags", middleware.RepoType(types.DatasetRepo), repoCommonHandler.Tags)
+ // update tags of a certain category
+ datasetsGroup.POST("/:namespace/:name/tags/:category", middleware.RepoType(types.DatasetRepo), repoCommonHandler.UpdateTags)
+ datasetsGroup.GET("/:namespace/:name/last_commit", middleware.RepoType(types.DatasetRepo), repoCommonHandler.LastCommit)
+ datasetsGroup.GET("/:namespace/:name/commit/:commit_id", middleware.RepoType(types.DatasetRepo), repoCommonHandler.CommitWithDiff)
+ datasetsGroup.GET("/:namespace/:name/tree", middleware.RepoType(types.DatasetRepo), repoCommonHandler.Tree)
+ datasetsGroup.GET("/:namespace/:name/commits", middleware.RepoType(types.DatasetRepo), repoCommonHandler.Commits)
+ datasetsGroup.POST("/:namespace/:name/raw/*file_path", middleware.RepoType(types.DatasetRepo), repoCommonHandler.CreateFile)
+ datasetsGroup.GET("/:namespace/:name/raw/*file_path", middleware.RepoType(types.DatasetRepo), repoCommonHandler.FileRaw)
+ datasetsGroup.GET("/:namespace/:name/blob/*file_path", middleware.RepoType(types.DatasetRepo), repoCommonHandler.FileInfo)
+ datasetsGroup.GET("/:namespace/:name/download/*file_path", middleware.RepoType(types.DatasetRepo), repoCommonHandler.DownloadFile)
+ datasetsGroup.GET("/:namespace/:name/resolve/*file_path", middleware.RepoType(types.DatasetRepo), repoCommonHandler.ResolveDownload)
+ datasetsGroup.PUT("/:namespace/:name/raw/*file_path", middleware.RepoType(types.DatasetRepo), repoCommonHandler.UpdateFile)
+ datasetsGroup.POST("/:namespace/:name/update_downloads", middleware.RepoType(types.DatasetRepo), repoCommonHandler.UpdateDownloads)
+ datasetsGroup.PUT("/:namespace/:name/incr_downloads", middleware.RepoType(types.DatasetRepo), repoCommonHandler.IncrDownloads)
+ datasetsGroup.POST("/:namespace/:name/upload_file", middleware.RepoType(types.DatasetRepo), repoCommonHandler.UploadFile)
+ datasetsGroup.POST("/:namespace/:name/mirror", middleware.RepoType(types.DatasetRepo), repoCommonHandler.CreateMirror)
+ datasetsGroup.GET("/:namespace/:name/mirror", middleware.RepoType(types.DatasetRepo), repoCommonHandler.GetMirror)
+ datasetsGroup.PUT("/:namespace/:name/mirror", middleware.RepoType(types.DatasetRepo), repoCommonHandler.UpdateMirror)
+ datasetsGroup.DELETE("/:namespace/:name/mirror", middleware.RepoType(types.DatasetRepo), repoCommonHandler.DeleteMirror)
+ datasetsGroup.POST("/:namespace/:name/mirror/sync", middleware.RepoType(types.DatasetRepo), repoCommonHandler.SyncMirror)
+ datasetsGroup.GET("/:namespace/:name/mirror/progress", middleware.RepoType(types.DatasetRepo), repoCommonHandler.MirrorProgress)
+
+ // mirror from SaaS, only on-premises available
+ if !config.Saas {
+ datasetsGroup.POST("/:namespace/:name/mirror_from_saas", middleware.RepoType(types.DatasetRepo), repoCommonHandler.MirrorFromSaas)
+ }
+
+ datasetsGroup.GET("/:namespace/:name/viewer/*file_path", dsViewerHandler.View)
+ datasetsGroup.GET("/:namespace/:name/dataviewer/catalog", dsViewerHandler.Catalog)
+ datasetsGroup.GET("/:namespace/:name/dataviewer/rows", dsViewerHandler.Rows)
+ }
+}
+
+func createCodeRoutes(config *config.Config, apiGroup *gin.RouterGroup, codeHandler *handler.CodeHandler, repoCommonHandler *handler.RepoHandler) {
+ codesGroup := apiGroup.Group("/codes")
+ {
+ codesGroup.POST("", codeHandler.Create)
+ codesGroup.GET("", codeHandler.Index)
+ codesGroup.PUT("/:namespace/:name", codeHandler.Update)
+ codesGroup.DELETE("/:namespace/:name", codeHandler.Delete)
+ codesGroup.GET("/:namespace/:name", codeHandler.Show)
+ codesGroup.GET("/:namespace/:name/relations", codeHandler.Relations)
+ codesGroup.GET("/:namespace/:name/branches", middleware.RepoType(types.CodeRepo), repoCommonHandler.Branches)
+ codesGroup.GET("/:namespace/:name/tags", middleware.RepoType(types.CodeRepo), repoCommonHandler.Tags)
+ // update tags of a certain category
+ codesGroup.POST("/:namespace/:name/tags/:category", middleware.RepoType(types.CodeRepo), repoCommonHandler.UpdateTags)
+ codesGroup.GET("/:namespace/:name/last_commit", middleware.RepoType(types.CodeRepo), repoCommonHandler.LastCommit)
+ codesGroup.GET("/:namespace/:name/commit/:commit_id", middleware.RepoType(types.CodeRepo), repoCommonHandler.CommitWithDiff)
+ codesGroup.GET("/:namespace/:name/tree", middleware.RepoType(types.CodeRepo), repoCommonHandler.Tree)
+ codesGroup.GET("/:namespace/:name/commits", middleware.RepoType(types.CodeRepo), repoCommonHandler.Commits)
+ codesGroup.POST("/:namespace/:name/raw/*file_path", middleware.RepoType(types.CodeRepo), repoCommonHandler.CreateFile)
+ codesGroup.GET("/:namespace/:name/raw/*file_path", middleware.RepoType(types.CodeRepo), repoCommonHandler.FileRaw)
+ codesGroup.GET("/:namespace/:name/blob/*file_path", middleware.RepoType(types.CodeRepo), repoCommonHandler.FileInfo)
+ codesGroup.GET("/:namespace/:name/download/*file_path", middleware.RepoType(types.CodeRepo), repoCommonHandler.DownloadFile)
+ codesGroup.GET("/:namespace/:name/resolve/*file_path", middleware.RepoType(types.CodeRepo), repoCommonHandler.ResolveDownload)
+ codesGroup.PUT("/:namespace/:name/raw/*file_path", middleware.RepoType(types.CodeRepo), repoCommonHandler.UpdateFile)
+ codesGroup.POST("/:namespace/:name/update_downloads", middleware.RepoType(types.CodeRepo), repoCommonHandler.UpdateDownloads)
+ codesGroup.PUT("/:namespace/:name/incr_downloads", middleware.RepoType(types.CodeRepo), repoCommonHandler.IncrDownloads)
+ codesGroup.POST("/:namespace/:name/upload_file", middleware.RepoType(types.CodeRepo), repoCommonHandler.UploadFile)
+ codesGroup.POST("/:namespace/:name/mirror", middleware.RepoType(types.CodeRepo), repoCommonHandler.CreateMirror)
+ codesGroup.GET("/:namespace/:name/mirror", middleware.RepoType(types.CodeRepo), repoCommonHandler.GetMirror)
+ codesGroup.PUT("/:namespace/:name/mirror", middleware.RepoType(types.CodeRepo), repoCommonHandler.UpdateMirror)
+ codesGroup.DELETE("/:namespace/:name/mirror", middleware.RepoType(types.CodeRepo), repoCommonHandler.DeleteMirror)
+ codesGroup.POST("/:namespace/:name/mirror/sync", middleware.RepoType(types.CodeRepo), repoCommonHandler.SyncMirror)
+ codesGroup.GET("/:namespace/:name/mirror/progress", middleware.RepoType(types.CodeRepo), repoCommonHandler.MirrorProgress)
+
+ // mirror from SaaS, only on-premises available
+ if !config.Saas {
+ codesGroup.POST("/:namespace/:name/mirror_from_saas", middleware.RepoType(types.CodeRepo), repoCommonHandler.MirrorFromSaas)
+ }
+ }
+}
+
+func createSpaceRoutes(config *config.Config, apiGroup *gin.RouterGroup, spaceHandler *handler.SpaceHandler, repoCommonHandler *handler.RepoHandler) {
+ spaces := apiGroup.Group("/spaces")
+ {
+ // list all spaces
+ spaces.GET("", spaceHandler.Index)
+ spaces.POST("", spaceHandler.Create)
+ // show a user or org's space
+ spaces.GET("/:namespace/:name", spaceHandler.Show)
+ spaces.PUT("/:namespace/:name", spaceHandler.Update)
+ spaces.DELETE("/:namespace/:name", spaceHandler.Delete)
+ // depoly and start running the space
+ spaces.POST("/:namespace/:name/run", spaceHandler.Run)
+ // wake a sleeping space
+ spaces.POST("/:namespace/:name/wakeup", spaceHandler.Wakeup)
+ // stop running space
+ spaces.POST("/:namespace/:name/stop", spaceHandler.Stop)
+ // pull space running status
+ spaces.GET("/:namespace/:name/status", spaceHandler.Status)
+ // pull space building and running logs
+ spaces.GET("/:namespace/:name/logs", spaceHandler.Logs)
+ // call space webhook api
+ spaces.POST("/:namespace/:name/webhook", nil)
+
+ spaces.GET("/:namespace/:name/branches", middleware.RepoType(types.SpaceRepo), repoCommonHandler.Branches)
+ spaces.GET("/:namespace/:name/tags", middleware.RepoType(types.SpaceRepo), repoCommonHandler.Tags)
+ // update tags of a certain category
+ spaces.POST("/:namespace/:name/tags/:category", middleware.RepoType(types.SpaceRepo), repoCommonHandler.UpdateTags)
+ spaces.GET("/:namespace/:name/last_commit", middleware.RepoType(types.SpaceRepo), repoCommonHandler.LastCommit)
+ spaces.GET("/:namespace/:name/commit/:commit_id", middleware.RepoType(types.SpaceRepo), repoCommonHandler.CommitWithDiff)
+ spaces.GET("/:namespace/:name/tree", middleware.RepoType(types.SpaceRepo), repoCommonHandler.Tree)
+ spaces.GET("/:namespace/:name/commits", middleware.RepoType(types.SpaceRepo), repoCommonHandler.Commits)
+ spaces.POST("/:namespace/:name/raw/*file_path", middleware.RepoType(types.SpaceRepo), repoCommonHandler.CreateFile)
+ spaces.GET("/:namespace/:name/raw/*file_path", middleware.RepoType(types.SpaceRepo), repoCommonHandler.FileRaw)
+ spaces.GET("/:namespace/:name/blob/*file_path", middleware.RepoType(types.SpaceRepo), repoCommonHandler.FileInfo)
+ spaces.GET("/:namespace/:name/download/*file_path", middleware.RepoType(types.SpaceRepo), repoCommonHandler.DownloadFile)
+ spaces.GET("/:namespace/:name/resolve/*file_path", middleware.RepoType(types.SpaceRepo), repoCommonHandler.ResolveDownload)
+ spaces.PUT("/:namespace/:name/raw/*file_path", middleware.RepoType(types.SpaceRepo), repoCommonHandler.UpdateFile)
+ spaces.POST("/:namespace/:name/update_downloads", middleware.RepoType(types.SpaceRepo), repoCommonHandler.UpdateDownloads)
+ spaces.PUT("/:namespace/:name/incr_downloads", middleware.RepoType(types.SpaceRepo), repoCommonHandler.IncrDownloads)
+ spaces.POST("/:namespace/:name/upload_file", middleware.RepoType(types.SpaceRepo), repoCommonHandler.UploadFile)
+ spaces.POST("/:namespace/:name/mirror", middleware.RepoType(types.SpaceRepo), repoCommonHandler.CreateMirror)
+ spaces.GET("/:namespace/:name/mirror", middleware.RepoType(types.SpaceRepo), repoCommonHandler.GetMirror)
+ spaces.PUT("/:namespace/:name/mirror", middleware.RepoType(types.SpaceRepo), repoCommonHandler.UpdateMirror)
+ spaces.DELETE("/:namespace/:name/mirror", middleware.RepoType(types.SpaceRepo), repoCommonHandler.DeleteMirror)
+ spaces.POST("/:namespace/:name/mirror/sync", middleware.RepoType(types.SpaceRepo), repoCommonHandler.SyncMirror)
+ spaces.GET("/:namespace/:name/mirror/progress", middleware.RepoType(types.SpaceRepo), repoCommonHandler.MirrorProgress)
+
+ // mirror from SaaS, only on-premises available
+ if !config.Saas {
+ spaces.POST("/:namespace/:name/mirror_from_saas", middleware.RepoType(types.SpaceRepo), repoCommonHandler.MirrorFromSaas)
+ }
+ spaces.GET("/:namespace/:name/run", middleware.RepoType(types.SpaceRepo), repoCommonHandler.DeployList)
+ spaces.GET("/:namespace/:name/run/:id", middleware.RepoType(types.SpaceRepo), repoCommonHandler.DeployDetail)
+ spaces.GET("/:namespace/:name/run/:id/status", middleware.RepoType(types.SpaceRepo), repoCommonHandler.DeployStatus)
+ spaces.GET("/:namespace/:name/run/:id/logs/:instance", middleware.RepoType(types.SpaceRepo), repoCommonHandler.DeployInstanceLogs)
+ }
+}
+
+func createUserRoutes(apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, userProxyHandler *handler.InternalServiceProxyHandler, userHandler *handler.UserHandler) {
+ // depricated
+ {
+ apiGroup.POST("/users", userProxyHandler.ProxyToApi("/api/v1/user"))
+ apiGroup.PUT("/users/:username", userProxyHandler.ProxyToApi("/api/v1/user/%v", "username"))
+ }
+
+ {
+ apiGroup.POST("/user", userProxyHandler.Proxy)
+ apiGroup.GET("/user/:username", userProxyHandler.Proxy)
+ apiGroup.PUT("/user/:username", userProxyHandler.Proxy)
+ apiGroup.DELETE("/user/:username", userProxyHandler.Proxy)
+ }
+
+ {
+ // User models
+ apiGroup.GET("/user/:username/models", userHandler.Models)
+ // User datasets
+ apiGroup.GET("/user/:username/datasets", userHandler.Datasets)
+ apiGroup.GET("/user/:username/codes", userHandler.Codes)
+ apiGroup.GET("/user/:username/spaces", userHandler.Spaces)
+ apiGroup.GET("/user/:username/prompts", userHandler.Prompts)
+ // User likes
+ apiGroup.PUT("/user/:username/likes/:repo_id", userHandler.LikesAdd)
+ apiGroup.DELETE("/user/:username/likes/:repo_id", userHandler.LikesDelete)
+ apiGroup.GET("/user/:username/likes/spaces", userHandler.LikesSpaces)
+ apiGroup.GET("/user/:username/likes/codes", userHandler.LikesCodes)
+ apiGroup.GET("/user/:username/likes/models", userHandler.LikesModels)
+ apiGroup.GET("/user/:username/likes/datasets", userHandler.LikesDatasets)
+ apiGroup.GET("/user/:username/run/:repo_type", userHandler.GetRunDeploys)
+ apiGroup.GET("/user/:username/finetune/instances", userHandler.GetFinetuneInstances)
+ // User evaluations
+ apiGroup.GET("/user/:username/evaluations", userHandler.GetEvaluations)
+ }
+
+ // User collection
+ apiGroup.GET("/user/:username/collections", userHandler.UserCollections)
+ apiGroup.GET("/user/:username/likes/collections", userHandler.LikesCollections)
+ apiGroup.PUT("/user/:username/likes/collections/:id", userHandler.LikeCollection)
+ apiGroup.DELETE("/user/:username/likes/collections/:id", userHandler.UnLikeCollection)
+ // user owned tokens
+ apiGroup.GET("/user/:username/tokens", userProxyHandler.ProxyToApi("/api/v1/user/%s/tokens", "username"))
+
+ // serverless list
+ apiGroup.GET("/user/:username/run/serverless", needAPIKey, userHandler.GetRunServerless)
+ // get user resource
+ apiGroup.GET("/user/:username/order/resources", userHandler.GetUserResource)
+ // cretate user resource
+ apiGroup.POST("/user/:username/order/resources", needAPIKey, userHandler.CreateUserResource)
+ // delete user resource
+ apiGroup.DELETE("/user/:username/order/resources/:id", needAPIKey, userHandler.DeleteUserResource)
+}
+
+func createRuntimeFrameworkRoutes(apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, modelHandler *handler.ModelHandler, runtimeArchHandler *handler.RuntimeArchitectureHandler) {
+ runtimeFramework := apiGroup.Group("/runtime_framework")
+ {
+ runtimeFramework.GET("/:id/models", modelHandler.ListByRuntimeFrameworkID)
+ runtimeFramework.GET("", modelHandler.ListAllRuntimeFramework)
+ runtimeFramework.POST("/:id", needAPIKey, modelHandler.UpdateModelRuntimeFrameworks)
+ runtimeFramework.DELETE("/:id", needAPIKey, modelHandler.DeleteModelRuntimeFrameworks)
+ runtimeFramework.GET("/models", modelHandler.ListModelsOfRuntimeFrameworks)
+
+ runtimeFramework.GET("/:id/architecture", needAPIKey, runtimeArchHandler.ListByRuntimeFrameworkID)
+ runtimeFramework.PUT("/:id/architecture", needAPIKey, runtimeArchHandler.UpdateArchitecture)
+ runtimeFramework.DELETE("/:id/architecture", needAPIKey, runtimeArchHandler.DeleteArchitecture)
+ runtimeFramework.POST("/:id/scan", needAPIKey, runtimeArchHandler.ScanArchitecture)
+ }
+}
+
+func createAccountRoutes(apiGroup *gin.RouterGroup, needAPIKey gin.HandlerFunc, accountingHandler *handler.AccountingHandler,
+ accountingProxyHandler *handler.InternalServiceProxyHandler) {
+
+ {
+ apiGroup.POST("/accounting/recharge/create-pay-order", accountingProxyHandler.Proxy)
+ apiGroup.GET("/accounting/recharge/:id/status", accountingProxyHandler.Proxy)
+ apiGroup.GET("/accounting/recharge/user-recharge-list", accountingProxyHandler.Proxy)
+ }
+
+ accountingGroup := apiGroup.Group("/accounting")
+ {
+ creditGroup := accountingGroup.Group("/credit")
+ {
+ creditGroup.GET("/balance", accountingHandler.QueryAllUsersBalance)
+ creditGroup.GET("/:id/balance", accountingHandler.QueryBalanceByUserID)
+ creditGroup.GET("/:id/statements", accountingHandler.QueryStatementByUserID)
+ creditGroup.GET("/:id/bills", accountingHandler.QueryBillsByUserID)
+ creditGroup.PUT("/:id/recharge", accountingHandler.RechargeByUserID)
+ }
+ multiSyncGroup := accountingGroup.Group("/multisync")
+ {
+ multiSyncGroup.POST("/quotas", accountingHandler.CreateOrUpdateQuota)
+ multiSyncGroup.GET("/quota", accountingHandler.QueryQuota)
+ multiSyncGroup.POST("/downloads", accountingHandler.CreateQuotaStatement)
+ multiSyncGroup.GET("/download", accountingHandler.QueryQuotaStatement)
+ }
+ meterGroup := accountingGroup.Group("/metering")
+ {
+ meterGroup.GET("/:id/statements", accountingHandler.QueryMeteringStatementByUserID)
+ }
+ priceGroup := accountingGroup.Group("/price")
+ {
+ priceGroup.POST("", needAPIKey, accountingHandler.PriceCreate)
+ priceGroup.GET("/:id", needAPIKey, accountingHandler.GetPriceByID)
+ priceGroup.PUT("/:id", needAPIKey, accountingHandler.PriceUpdate)
+ priceGroup.DELETE("/:id", needAPIKey, accountingHandler.PriceDelete)
+ priceGroup.GET("", needAPIKey, accountingHandler.QueryPricesBySKUType)
+ }
+ }
+}
+
+func createHFRoutes(r *gin.Engine, hfdsHandler *handler.HFDatasetHandler, repoCommonHandler *handler.RepoHandler, modelHandler *handler.ModelHandler, userHandler *handler.UserHandler) {
+ // Huggingface SDK routes
+ hfGroup := r.Group("/hf")
+ {
+ hfGroup.GET("/:namespace/:name/resolve/:branch/*file_path", middleware.RepoMapping(types.ModelRepo), repoCommonHandler.SDKDownload)
+ hfGroup.HEAD("/:namespace/:name/resolve/:branch/*file_path", middleware.RepoMapping(types.ModelRepo), repoCommonHandler.HeadSDKDownload)
+ hfdsFileGroup := hfGroup.Group("/datasets")
+ {
+ hfdsFileGroup.GET("/:namespace/:name/resolve/:branch/*file_path", middleware.RepoMapping(types.DatasetRepo), repoCommonHandler.SDKDownload)
+ hfdsFileGroup.HEAD("/:namespace/:name/resolve/:branch/*file_path", middleware.RepoMapping(types.DatasetRepo), repoCommonHandler.HeadSDKDownload)
+ }
+ hfAPIGroup := hfGroup.Group("/api")
+ {
+ hfAPIGroup.GET("/whoami-v2", userHandler.UserPermission)
+ hfModelAPIGroup := hfAPIGroup.Group("/models")
+ {
+ // compatible with HF model info api, used for sdk like this: huggingface_hub.model_info(repo_id, revision)
+ hfModelAPIGroup.GET("/:namespace/:name/revision/:ref", middleware.RepoMapping(types.ModelRepo), modelHandler.SDKModelInfo)
+ hfModelAPIGroup.GET("/:namespace/:name", middleware.RepoMapping(types.ModelRepo), modelHandler.SDKModelInfo)
+ }
+ hfDSAPIGroup := hfAPIGroup.Group("/datasets")
+ {
+ // compatible with HF dataset info api, used for sdk like this: huggingface_hub.dataset_info(repo_id, revision)
+ hfDSAPIGroup.GET("/:namespace/:name/revision/:ref", middleware.RepoMapping(types.DatasetRepo), repoCommonHandler.SDKListFiles)
+ hfDSAPIGroup.GET("/:namespace/:name", middleware.RepoMapping(types.DatasetRepo), repoCommonHandler.SDKListFiles)
+ hfDSAPIGroup.POST("/:namespace/:name/paths-info/:ref", middleware.RepoMapping(types.DatasetRepo), hfdsHandler.DatasetPathsInfo)
+ hfDSAPIGroup.GET("/:namespace/:name/tree/:ref/*path_in_repo", middleware.RepoMapping(types.DatasetRepo), hfdsHandler.DatasetTree)
+ hfDSAPIGroup.GET("/:namespace/:name/resolve/:ref/.huggingface.yaml", middleware.RepoMapping(types.DatasetRepo), hfdsHandler.HandleHFYaml)
+ }
+ }
+ }
+
+}
+
+func createDiscussionRoutes(apiGroup *gin.RouterGroup, discussionHandler *handler.DiscussionHandler) {
+ apiGroup.POST("/:repo_type/:namespace/:name/discussions", discussionHandler.CreateRepoDiscussion)
+ apiGroup.GET("/:repo_type/:namespace/:name/discussions", discussionHandler.ListRepoDiscussions)
+ apiGroup.GET("/discussions/:id", discussionHandler.ShowDiscussion)
+ apiGroup.PUT("/discussions/:id", discussionHandler.UpdateDiscussion)
+ apiGroup.DELETE("/discussions/:id", discussionHandler.DeleteDiscussion)
+ apiGroup.POST("/discussions/:id/comments", discussionHandler.CreateDiscussionComment)
+ apiGroup.GET("/discussions/:id/comments", discussionHandler.ListDiscussionComments)
+ apiGroup.PUT("/discussions/:id/comments/:comment_id", discussionHandler.UpdateComment)
+ apiGroup.DELETE("/discussions/:id/comments/:comment_id", discussionHandler.DeleteComment)
+}
+
+func createPromptRoutes(apiGroup *gin.RouterGroup, promptHandler *handler.PromptHandler) {
+ promptGrp := apiGroup.Group("/prompts")
+ {
+ promptGrp.GET("", promptHandler.Index)
+ promptGrp.GET("/:namespace/:name", promptHandler.ListPrompt)
+ promptGrp.GET("/:namespace/:name/relations", promptHandler.Relations)
+ promptGrp.GET("/:namespace/:name/prompt/view/*file_path", promptHandler.GetPrompt)
+ promptGrp.POST("/:namespace/:name/prompt/record", promptHandler.CreatePrompt)
+ promptGrp.PUT("/:namespace/:name/prompt/record/*file_path", promptHandler.UpdatePrompt)
+ promptGrp.DELETE("/:namespace/:name/prompt/record/*file_path", promptHandler.DeletePrompt)
+
+ promptGrp.PUT("/:namespace/:name/relations", promptHandler.SetRelations)
+ promptGrp.POST("/:namespace/:name/relations/model", promptHandler.AddModelRelation)
+ promptGrp.DELETE("/:namespace/:name/relations/model", promptHandler.DelModelRelation)
+ conversationGrp := promptGrp.Group("/conversations")
+ {
+ conversationGrp.POST("", promptHandler.NewConversation)
+ conversationGrp.GET("", promptHandler.ListConversation)
+ conversationGrp.GET("/:id", promptHandler.GetConversation)
+ conversationGrp.POST("/:id", promptHandler.SubmitMessage)
+ conversationGrp.PUT("/:id", promptHandler.UpdateConversation)
+ conversationGrp.DELETE("/:id", promptHandler.RemoveConversation)
+ conversationGrp.PUT("/:id/message/:msgid/like", promptHandler.LikeMessage)
+ conversationGrp.PUT("/:id/message/:msgid/hate", promptHandler.HateMessage)
+ }
+
+ promptGrp.POST("", promptHandler.Create)
+ promptGrp.PUT("/:namespace/:name", promptHandler.Update)
+ promptGrp.DELETE("/:namespace/:name", promptHandler.Delete)
+
+ promptGrp.GET("/:namespace/:name/branches", promptHandler.Branches)
+ promptGrp.GET("/:namespace/:name/tags", promptHandler.Tags)
+ promptGrp.POST("/:namespace/:name/tags/:category", promptHandler.UpdateTags)
+ promptGrp.POST("/:namespace/:name/update_downloads", promptHandler.UpdateDownloads)
+ }
+}
+
+func createDataflowRoutes(apiGroup *gin.RouterGroup, dataflowHandler *handler.DataflowProxyHandler) {
+ dataflowGrp := apiGroup.Group("/dataflow")
+ dataflowGrp.Use(middleware.MustLogin())
+ {
+ jobGrp := dataflowGrp.Group("/jobs")
+ {
+ jobGrp.GET("", dataflowHandler.Proxy)
+ jobGrp.POST("", dataflowHandler.Proxy)
+ jobGrp.GET("/:id", dataflowHandler.Proxy)
+ jobGrp.DELETE("/:id", dataflowHandler.Proxy)
+ jobGrp.GET("/log/:id", dataflowHandler.Proxy)
+ jobGrp.GET("/search", dataflowHandler.Proxy)
+ }
+ templateGrp := dataflowGrp.Group("/templates")
+ {
+ templateGrp.GET("", dataflowHandler.Proxy)
+ templateGrp.POST("", dataflowHandler.Proxy)
+ templateGrp.DELETE("/:template_id", dataflowHandler.Proxy)
+ }
+ opsGrp := dataflowGrp.Group("/ops")
+ {
+ opsGrp.GET("", dataflowHandler.Proxy)
+ }
+ toolGrp := dataflowGrp.Group("/tools")
+ {
+ toolGrp.GET("", dataflowHandler.Proxy)
+ }
+ }
+}
+
+
+
//go:build !ee && !saas
+
+package router
+
+import (
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/handler"
+)
+
+func createLicenseRoutes(apiGroup *gin.RouterGroup, licenseHandler *handler.LicenseHandler) {
+
+}
+
+
+
package router
+
+import (
+ "fmt"
+ "net/http"
+
+ "github.com/gin-contrib/cors"
+ "github.com/gin-contrib/sessions"
+ "github.com/gin-contrib/sessions/cookie"
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/handler"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func NewRProxyRouter(config *config.Config) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(cors.New(cors.Config{
+ AllowCredentials: true,
+ AllowHeaders: []string{"*"},
+ AllowMethods: []string{"*"},
+ AllowAllOrigins: true,
+ }))
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+ store := cookie.NewStore([]byte(config.Space.SessionSecretKey))
+ store.Options(sessions.Options{
+ SameSite: http.SameSiteNoneMode,
+ Secure: config.EnableHTTPS,
+ })
+ r.Use(sessions.Sessions("jwt_session", store))
+ //to access space with jwt token in query string
+ r.Use(middleware.BuildJwtSession(config.JWT.SigningKey))
+ //to access model,fintune with any kind of tokens in auth header
+ r.Use(middleware.Authenticator(config))
+
+ handler, err := handler.NewRProxyHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating rproxy handler:%w", err)
+ }
+ r.Any("/*api", middleware.AuthSession(), handler.Proxy)
+
+ return r, nil
+}
+
+
+
package activity
+
+import (
+ "context"
+ "log/slog"
+
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+func CalcRecomScore(ctx context.Context, config *config.Config) error {
+ c, err := component.NewRecomComponent(config)
+ if err != nil {
+ slog.Error("failed to create recom component", "err", err)
+ return err
+ }
+ c.CalculateRecomScore(context.Background())
+ return nil
+}
+
+
+
package activity
+
+import (
+ "context"
+ "fmt"
+
+ "go.temporal.io/sdk/activity"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component/callback"
+)
+
+func WatchSpaceChange(ctx context.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("watch space change start", "req", req)
+ callbackComponent, err := callback.NewGitCallback(config)
+ if err != nil {
+ return fmt.Errorf("failed to create callback component, error: %w", err)
+ }
+ callbackComponent.SetRepoVisibility(true)
+ return callbackComponent.WatchSpaceChange(ctx, req)
+}
+
+func WatchRepoRelation(ctx context.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("watch repo relation start", "req", req)
+ callbackComponent, err := callback.NewGitCallback(config)
+ if err != nil {
+ return fmt.Errorf("failed to create callback component, error: %w", err)
+ }
+ callbackComponent.SetRepoVisibility(true)
+ return callbackComponent.WatchRepoRelation(ctx, req)
+}
+
+func GenSyncVersion(ctx context.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("generate sync version start", "req", req)
+ callbackComponent, err := callback.NewGitCallback(config)
+ if err != nil {
+ return fmt.Errorf("failed to create callback component, error: %w", err)
+ }
+ callbackComponent.SetRepoVisibility(true)
+ return callbackComponent.GenSyncVersion(ctx, req)
+}
+
+func SetRepoUpdateTime(ctx context.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("set repo update time start", "req", req)
+ callbackComponent, err := callback.NewGitCallback(config)
+ if err != nil {
+ return fmt.Errorf("failed to create callback component, error: %w", err)
+ }
+ callbackComponent.SetRepoVisibility(true)
+ return callbackComponent.SetRepoUpdateTime(ctx, req)
+}
+
+func UpdateRepoInfos(ctx context.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("update repo infos start", "req", req)
+ callbackComponent, err := callback.NewGitCallback(config)
+ if err != nil {
+ return fmt.Errorf("failed to create callback component, error: %w", err)
+ }
+ callbackComponent.SetRepoVisibility(true)
+ return callbackComponent.UpdateRepoInfos(ctx, req)
+}
+
+func SensitiveCheck(ctx context.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("sensitive check start", "req", req)
+ callbackComponent, err := callback.NewGitCallback(config)
+ if err != nil {
+ return fmt.Errorf("failed to create callback component, error: %w", err)
+ }
+ callbackComponent.SetRepoVisibility(true)
+ return callbackComponent.SensitiveCheck(ctx, req)
+}
+
+
+
package activity
+
+import (
+ "context"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/multisync"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+func SyncAsClient(ctx context.Context, config *config.Config) error {
+ c, err := component.NewMultiSyncComponent(config)
+ if err != nil {
+ slog.Error("failed to create multi sync component", "err", err)
+ return err
+ }
+ syncClientSettingStore := database.NewSyncClientSettingStore()
+ setting, err := syncClientSettingStore.First(ctx)
+ if err != nil {
+ slog.Error("failed to find sync client setting", "error", err)
+ return err
+ }
+ apiDomain := config.MultiSync.SaasAPIDomain
+ sc := multisync.FromOpenCSG(apiDomain, setting.Token)
+ return c.SyncAsClient(ctx, sc)
+}
+
+
+
package workflow
+
+import (
+ "time"
+
+ "go.temporal.io/sdk/temporal"
+ "go.temporal.io/sdk/workflow"
+ "opencsg.com/csghub-server/api/workflow/activity"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func CalcRecomScoreWorkflow(ctx workflow.Context, config *config.Config) error {
+ logger := workflow.GetLogger(ctx)
+ logger.Info("calc recom score workflow started")
+
+ retryPolicy := &temporal.RetryPolicy{
+ MaximumAttempts: 3,
+ }
+
+ options := workflow.ActivityOptions{
+ StartToCloseTimeout: time.Hour * 1,
+ RetryPolicy: retryPolicy,
+ }
+
+ ctx = workflow.WithActivityOptions(ctx, options)
+ err := workflow.ExecuteActivity(ctx, activity.CalcRecomScore, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to calc recom score", "error", err)
+ return err
+ }
+ return nil
+}
+
+
+
package workflow
+
+import (
+ "time"
+
+ "go.temporal.io/sdk/temporal"
+ "go.temporal.io/sdk/workflow"
+ "opencsg.com/csghub-server/api/workflow/activity"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func SyncAsClientWorkflow(ctx workflow.Context, config *config.Config) error {
+ logger := workflow.GetLogger(ctx)
+ logger.Info("sync as client workflow started")
+
+ retryPolicy := &temporal.RetryPolicy{
+ MaximumAttempts: 3,
+ }
+
+ options := workflow.ActivityOptions{
+ StartToCloseTimeout: time.Hour * 1,
+ RetryPolicy: retryPolicy,
+ }
+
+ ctx = workflow.WithActivityOptions(ctx, options)
+ err := workflow.ExecuteActivity(ctx, activity.SyncAsClient, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to sync as client", "error", err)
+ return err
+ }
+ return nil
+}
+
+
+
package workflow
+
+import (
+ "context"
+ "fmt"
+
+ enumspb "go.temporal.io/api/enums/v1"
+ "go.temporal.io/sdk/client"
+ "go.temporal.io/sdk/worker"
+ "opencsg.com/csghub-server/api/workflow/activity"
+ "opencsg.com/csghub-server/common/config"
+)
+
+const (
+ AlreadyScheduledMessage = "schedule with this ID is already registered"
+ CronJobQueueName = "workflow_cron_queue"
+)
+
+func RegisterCronJobs(config *config.Config) error {
+ var err error
+ if wfClient == nil {
+ wfClient, err = client.Dial(client.Options{
+ HostPort: config.WorkFLow.Endpoint,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to create workflow client, error:%w", err)
+ }
+ }
+
+ if !config.Saas {
+ _, err = wfClient.ScheduleClient().Create(context.Background(), client.ScheduleOptions{
+ ID: "sync-as-client-schedule",
+ Spec: client.ScheduleSpec{
+ CronExpressions: []string{config.CronJob.SyncAsClientCronExpression},
+ },
+ Overlap: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP,
+ Action: &client.ScheduleWorkflowAction{
+ ID: "sync-as-client-workflow",
+ TaskQueue: CronJobQueueName,
+ Workflow: SyncAsClientWorkflow,
+ Args: []interface{}{config},
+ },
+ })
+ if err != nil && err.Error() != AlreadyScheduledMessage {
+ return fmt.Errorf("unable to create schedule, error:%w", err)
+ }
+ }
+
+ _, err = wfClient.ScheduleClient().Create(context.Background(), client.ScheduleOptions{
+ ID: "calc-recom-score-schedule",
+ Spec: client.ScheduleSpec{
+ CronExpressions: []string{config.CronJob.CalcRecomScoreCronExpression},
+ },
+ Overlap: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP,
+ Action: &client.ScheduleWorkflowAction{
+ ID: "calc-recom-score-workflow",
+ TaskQueue: CronJobQueueName,
+ Workflow: CalcRecomScoreWorkflow,
+ Args: []interface{}{config},
+ },
+ })
+ if err != nil && err.Error() != AlreadyScheduledMessage {
+ return fmt.Errorf("unable to create schedule, error:%w", err)
+ }
+
+ return nil
+}
+
+func StartCronWorker(config *config.Config) error {
+ var err error
+ if wfClient == nil {
+ wfClient, err = client.Dial(client.Options{
+ HostPort: config.WorkFLow.Endpoint,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to create workflow client, error:%w", err)
+ }
+ }
+ wfWorker = worker.New(wfClient, CronJobQueueName, worker.Options{})
+ if !config.Saas {
+ wfWorker.RegisterWorkflow(SyncAsClientWorkflow)
+ wfWorker.RegisterActivity(activity.SyncAsClient)
+ }
+ wfWorker.RegisterWorkflow(CalcRecomScoreWorkflow)
+ wfWorker.RegisterActivity(activity.CalcRecomScore)
+
+ return wfWorker.Start()
+}
+
+
+
package workflow
+
+import (
+ "time"
+
+ "go.temporal.io/sdk/temporal"
+ "go.temporal.io/sdk/workflow"
+
+ "opencsg.com/csghub-server/api/workflow/activity"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func HandlePushWorkflow(ctx workflow.Context, req *types.GiteaCallbackPushReq, config *config.Config) error {
+ logger := workflow.GetLogger(ctx)
+ logger.Info("handle push workflow started")
+
+ retryPolicy := &temporal.RetryPolicy{
+ MaximumAttempts: 3,
+ }
+
+ options := workflow.ActivityOptions{
+ StartToCloseTimeout: time.Hour * 1,
+ RetryPolicy: retryPolicy,
+ }
+
+ ctx = workflow.WithActivityOptions(ctx, options)
+ // Watch space change
+ err := workflow.ExecuteActivity(ctx, activity.WatchSpaceChange, req, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to watch space change", "error", err, "req", req)
+ return err
+ }
+
+ // Watch repo relation
+ err = workflow.ExecuteActivity(ctx, activity.WatchRepoRelation, req, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to watch repo relation", "error", err, "req", req)
+ return err
+ }
+
+ // Generate sync versions
+ err = workflow.ExecuteActivity(ctx, activity.GenSyncVersion, req, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to generate sync versions", "error", err, "req", req)
+ return err
+ }
+
+ // Set repo update time
+ err = workflow.ExecuteActivity(ctx, activity.SetRepoUpdateTime, req, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to set repo update time", "error", err, "req", req)
+ return err
+ }
+
+ // Update repo infos
+ err = workflow.ExecuteActivity(ctx, activity.UpdateRepoInfos, req, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to update repo infos", "error", err, "req", req)
+ return err
+ }
+
+ // Sensitive check
+ err = workflow.ExecuteActivity(ctx, activity.SensitiveCheck, req, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to sensitive check", "error", err, "req", req)
+ return err
+ }
+
+ return nil
+}
+
+
+
package workflow
+
+import (
+ "fmt"
+
+ "go.temporal.io/sdk/client"
+ "go.temporal.io/sdk/worker"
+ "opencsg.com/csghub-server/api/workflow/activity"
+ "opencsg.com/csghub-server/common/config"
+)
+
+const HandlePushQueueName = "workflow_handle_push_queue"
+
+var (
+ wfWorker worker.Worker
+ wfClient client.Client
+)
+
+func StartWorker(config *config.Config) error {
+ var err error
+ wfClient, err = client.Dial(client.Options{
+ HostPort: config.WorkFLow.Endpoint,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to create workflow client, error:%w", err)
+ }
+ wfWorker = worker.New(wfClient, HandlePushQueueName, worker.Options{})
+ wfWorker.RegisterWorkflow(HandlePushWorkflow)
+ wfWorker.RegisterActivity(activity.WatchSpaceChange)
+ wfWorker.RegisterActivity(activity.WatchRepoRelation)
+ wfWorker.RegisterActivity(activity.GenSyncVersion)
+ wfWorker.RegisterActivity(activity.SetRepoUpdateTime)
+ wfWorker.RegisterActivity(activity.UpdateRepoInfos)
+ wfWorker.RegisterActivity(activity.SensitiveCheck)
+
+ return wfWorker.Start()
+}
+
+func StopWorker() {
+ if wfWorker != nil {
+ wfWorker.Stop()
+ }
+ if wfClient != nil {
+ wfClient.Close()
+ }
+}
+
+func GetWorkflowClient() client.Client {
+ return wfClient
+}
+
+
+
package accounting
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type AccountingClient interface {
+ QueryAllUsersBalance(per, page int) (any, error)
+ QueryBalanceByUserID(userUUID string) (any, error)
+ ListStatementByUserIDAndTime(req types.ACCT_STATEMENTS_REQ) (any, error)
+ ListBillsByUserIDAndDate(req types.ACCT_STATEMENTS_REQ) (any, error)
+ RechargeAccountingUser(userID string, req types.RECHARGE_REQ) (any, error)
+ PresentAccountingUser(userID string, req types.ACTIVITY_REQ) (any, error)
+ CreateOrUpdateQuota(currentUser string, req types.ACCT_QUOTA_REQ) (any, error)
+ GetQuotaByID(currentUser string) (any, error)
+ CreateQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (any, error)
+ GetQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (any, error)
+ QueryPricesBySKUType(currentUser string, req types.AcctPriceListReq) (any, error)
+ GetPriceByID(currentUser string, id int64) (any, error)
+ CreatePrice(currentUser string, req types.AcctPriceCreateReq) (any, error)
+ UpdatePrice(currentUser string, req types.AcctPriceCreateReq, id int64) (any, error)
+ DeletePrice(currentUser string, id int64) (any, error)
+ ListMeteringsByUserIDAndTime(req types.ACCT_STATEMENTS_REQ) (any, error)
+ CreateOrder(currentUser string, req types.AcctOrderCreateReq) (any, error)
+}
+type accountingClientImpl struct {
+ remote *url.URL
+ client *http.Client
+ authToken string
+}
+
+func NewAccountingClient(config *config.Config) (*accountingClientImpl, error) {
+ remoteURL := fmt.Sprintf("%s:%d", config.Accounting.Host, config.Accounting.Port)
+ parsedURL, err := url.Parse(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ return &accountingClientImpl{
+ remote: parsedURL,
+ client: http.DefaultClient,
+ authToken: config.APIToken,
+ }, nil
+}
+
+func (ac *accountingClientImpl) QueryAllUsersBalance(per, page int) (any, error) {
+ subUrlPath := fmt.Sprintf("/credit/balance?per=%d&page=%d", per, page)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) QueryBalanceByUserID(userUUID string) (any, error) {
+ subUrlPath := fmt.Sprintf("/credit/%s/balance", userUUID)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) ListStatementByUserIDAndTime(req types.ACCT_STATEMENTS_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/credit/%s/statements?current_user=%s&scene=%d&instance_name=%s&start_time=%s&end_time=%s&per=%d&page=%d", req.UserUUID, req.CurrentUser, req.Scene, req.InstanceName, url.QueryEscape(req.StartTime), url.QueryEscape(req.EndTime), req.Per, req.Page)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) ListBillsByUserIDAndDate(req types.ACCT_STATEMENTS_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/credit/%s/bills?current_user=%s&scene=%d&start_date=%s&end_date=%s&per=%d&page=%d", req.UserUUID, req.CurrentUser, req.Scene, url.QueryEscape(req.StartTime), url.QueryEscape(req.EndTime), req.Per, req.Page)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) RechargeAccountingUser(userID string, req types.RECHARGE_REQ) (any, error) {
+ // Todo: check super admin to do this action
+ subUrlPath := fmt.Sprintf("/credit/%s/recharge?current_user=%s", userID, userID)
+ return ac.handleResponse(ac.doRequest(http.MethodPut, subUrlPath, req))
+}
+
+func (ac *accountingClientImpl) PresentAccountingUser(userID string, req types.ACTIVITY_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/credit/%s/present", userID)
+ return ac.handleResponse(ac.doRequest(http.MethodPut, subUrlPath, req))
+}
+
+func (ac *accountingClientImpl) CreateOrUpdateQuota(currentUser string, req types.ACCT_QUOTA_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/multisync/quotas?current_user=%s", currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodPost, subUrlPath, req))
+}
+
+func (ac *accountingClientImpl) GetQuotaByID(currentUser string) (any, error) {
+ subUrlPath := fmt.Sprintf("/multisync/quota?current_user=%s", currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) CreateQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/multisync/downloads?current_user=%s", currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodPost, subUrlPath, req))
+}
+
+func (ac *accountingClientImpl) GetQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/multisync/download?current_user=%s&repo_path=%s&repo_type=%s", currentUser, req.RepoPath, req.RepoType)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) QueryPricesBySKUType(currentUser string, req types.AcctPriceListReq) (any, error) {
+ subUrlPath := fmt.Sprintf("/price?current_user=%s&sku_type=%d&sku_kind=%s&resource_id=%s&per=%d&page=%d", currentUser, req.SkuType, req.SkuKind, req.ResourceID, req.Per, req.Page)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) GetPriceByID(currentUser string, id int64) (any, error) {
+ subUrlPath := fmt.Sprintf("/price/%d?current_user=%s", id, currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) CreatePrice(currentUser string, req types.AcctPriceCreateReq) (any, error) {
+ subUrlPath := fmt.Sprintf("/price?current_user=%s", currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodPost, subUrlPath, req))
+}
+
+func (ac *accountingClientImpl) UpdatePrice(currentUser string, req types.AcctPriceCreateReq, id int64) (any, error) {
+ subUrlPath := fmt.Sprintf("/price/%d?current_user=%s", id, currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodPut, subUrlPath, req))
+}
+
+func (ac *accountingClientImpl) DeletePrice(currentUser string, id int64) (any, error) {
+ subUrlPath := fmt.Sprintf("/price/%d?current_user=%s", id, currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodDelete, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) ListMeteringsByUserIDAndTime(req types.ACCT_STATEMENTS_REQ) (any, error) {
+ subUrlPath := fmt.Sprintf("/metering/%s/statements?current_user=%s&scene=%d&instance_name=%s&start_time=%s&end_time=%s&per=%d&page=%d", req.UserUUID, req.CurrentUser, req.Scene, req.InstanceName, url.QueryEscape(req.StartTime), url.QueryEscape(req.EndTime), req.Per, req.Page)
+ return ac.handleResponse(ac.doRequest(http.MethodGet, subUrlPath, nil))
+}
+
+func (ac *accountingClientImpl) CreateOrder(currentUser string, req types.AcctOrderCreateReq) (any, error) {
+ subUrlPath := fmt.Sprintf("/order?current_user=%s", currentUser)
+ return ac.handleResponse(ac.doRequest(http.MethodPost, subUrlPath, req))
+}
+
+// Helper method to execute the actual HTTP request and read the response.
+func (ac *accountingClientImpl) doRequest(method, subPath string, data any) (*http.Response, error) {
+ urlPath := fmt.Sprintf("%s%s%s", ac.remote, "/api/v1/accounting", subPath)
+ // slog.Info("call", slog.Any("urlPath", urlPath))
+ var buf io.Reader
+ if data != nil {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ buf = bytes.NewBuffer(jsonData)
+ }
+
+ req, err := http.NewRequest(method, urlPath, buf)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Bearer "+ac.authToken)
+
+ resp, err := ac.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ var errData any
+ err := json.NewDecoder(resp.Body).Decode(&errData)
+ if err != nil {
+ return nil, fmt.Errorf("unexpected http status code: %d, %w", resp.StatusCode, err)
+ } else {
+ return nil, fmt.Errorf("unexpected http status and error: %d, %v", resp.StatusCode, errData)
+ }
+ }
+
+ return resp, nil
+}
+
+func (ac *accountingClientImpl) handleResponse(response *http.Response, err error) (any, error) {
+ if err != nil {
+ return nil, err
+ }
+ if response != nil && response.Body != nil {
+ defer response.Body.Close()
+ }
+ var res struct {
+ Msg string `json:"msg"`
+ Data any `json:"data"`
+ }
+ err = json.NewDecoder(response.Body).Decode(&res)
+ if err != nil {
+ return nil, err
+ }
+ return res.Data, nil
+}
+
+
+
package cluster
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "math"
+ "math/rand"
+ "path/filepath"
+ "strings"
+
+ "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
+ v1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/client-go/util/homedir"
+ knative "knative.dev/serving/pkg/client/clientset/versioned"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+// Cluster holds basic information about a Kubernetes cluster
+type Cluster struct {
+ ID string // Unique identifier for the cluster
+ ConfigPath string // Path to the kubeconfig file
+ Client *kubernetes.Clientset // Kubernetes client
+ KnativeClient *knative.Clientset // Knative client
+ ArgoClient *versioned.Clientset // Argo client
+ StorageClass string
+}
+
+// ClusterPool is a resource pool of cluster information
+type ClusterPool struct {
+ Clusters []Cluster
+ ClusterStore database.ClusterInfoStore
+}
+
+// NewClusterPool initializes and returns a ClusterPool by reading kubeconfig files from $HOME/.kube directory
+func NewClusterPool() (*ClusterPool, error) {
+ pool := &ClusterPool{}
+ pool.ClusterStore = database.NewClusterInfoStore()
+
+ home := homedir.HomeDir()
+ kubeconfigFolderPath := filepath.Join(home, ".kube")
+ kubeconfigFiles, err := filepath.Glob(filepath.Join(kubeconfigFolderPath, "config*"))
+ if err != nil {
+ return nil, err
+ }
+
+ if len(kubeconfigFiles) == 0 {
+ slog.Error("No kubeconfig files", slog.Any("path", kubeconfigFolderPath))
+ }
+
+ for i, kubeconfig := range kubeconfigFiles {
+ config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+ if err != nil {
+ return nil, err
+ }
+ client, err := kubernetes.NewForConfig(config)
+ if err != nil {
+ return nil, err
+ }
+ argoClient, err := versioned.NewForConfig(config)
+ if err != nil {
+ return nil, err
+ }
+ knativeClient, err := knative.NewForConfig(config)
+ if err != nil {
+ slog.Error("falied to create knative client", "error", err)
+ return nil, fmt.Errorf("falied to create knative client,%w", err)
+ }
+ id := filepath.Base(kubeconfig)
+ pool.Clusters = append(pool.Clusters, Cluster{
+ ID: id,
+ ConfigPath: kubeconfig,
+ Client: client,
+ KnativeClient: knativeClient,
+ ArgoClient: argoClient,
+ })
+ err = pool.ClusterStore.Add(context.TODO(), id, fmt.Sprintf("region-%d", i))
+ if err != nil {
+ slog.Error("falied to add cluster info to db", "error", err)
+ return nil, fmt.Errorf("falied to add cluster info to db,%w", err)
+ }
+ }
+
+ return pool, nil
+}
+
+// SelectCluster selects the most appropriate cluster to deploy the service to
+func (p *ClusterPool) GetCluster() (*Cluster, error) {
+ if len(p.Clusters) == 0 {
+ return nil, fmt.Errorf("no available clusters")
+ }
+ // Randomly choose a cluster to deploy the service to
+ // to do: The cluster should be selected based on criteria such as availability, performance, load, etc.
+ randomIndex := rand.Intn(len(p.Clusters))
+
+ // Select a cluster using the random index
+ selectedCluster := p.Clusters[randomIndex]
+ return &selectedCluster, nil
+}
+
+// GetClusterByID retrieves a cluster from the pool given its unique ID
+func (p *ClusterPool) GetClusterByID(ctx context.Context, id string) (*Cluster, error) {
+ cfId := "config"
+ storageClass := ""
+ if len(id) != 0 {
+ cInfo, _ := p.ClusterStore.ByClusterID(ctx, id)
+ cfId = cInfo.ClusterConfig
+ storageClass = cInfo.StorageClass
+ }
+ for _, Cluster := range p.Clusters {
+ if Cluster.ID == cfId {
+ Cluster.StorageClass = storageClass
+ return &Cluster, nil
+ }
+ }
+ return nil, fmt.Errorf("cluster with the given ID does not exist")
+}
+
+// getNodeResources retrieves all node cpu and gpu info
+func GetNodeResources(clientset *kubernetes.Clientset, config *config.Config) (map[string]types.NodeResourceInfo, error) {
+ nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+ if err != nil {
+ return nil, err
+ }
+
+ pods, err := clientset.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{})
+ if err != nil {
+ return nil, err
+ }
+
+ nodeResourcesMap := make(map[string]types.NodeResourceInfo)
+
+ for _, node := range nodes.Items {
+ memCapacity := node.Status.Capacity["memory"]
+ memQuantity, ok := memCapacity.AsInt64()
+ if !ok {
+ slog.Error("falied to get node memory", "node", node.Name, "error", err)
+ continue
+ }
+ totalMem := getMem(memQuantity)
+ totalCPU := node.Status.Capacity.Cpu().MilliValue()
+ totalXPU := resource.Quantity{}
+ xpuCapacityLabel, xpuTypeLabel := getXPULabel(node, config)
+ if xpuCapacityLabel != "" {
+ totalXPU = node.Status.Capacity[v1.ResourceName(xpuCapacityLabel)]
+ }
+
+ gpuModelVendor := strings.Split(node.Labels[xpuTypeLabel], "-")
+ gpuModel := ""
+ if len(gpuModelVendor) > 1 {
+ gpuModel = gpuModelVendor[1]
+ }
+ nodeResourcesMap[node.Name] = types.NodeResourceInfo{
+ NodeName: node.Name,
+ TotalCPU: millicoresToCores(totalCPU),
+ AvailableCPU: millicoresToCores(totalCPU),
+ XPUModel: gpuModel,
+ GPUVendor: gpuModelVendor[0],
+ TotalXPU: parseQuantityToInt64(totalXPU),
+ AvailableXPU: parseQuantityToInt64(totalXPU),
+ AvailableMem: totalMem,
+ TotalMem: totalMem,
+ XPUCapacityLabel: xpuCapacityLabel,
+ }
+ }
+ for _, pod := range pods.Items {
+ if pod.Spec.NodeName == "" || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed {
+ continue
+ }
+
+ nodeResource := nodeResourcesMap[pod.Spec.NodeName]
+ for _, container := range pod.Spec.Containers {
+ if requestedGPU, hasGPU := container.Resources.Requests[v1.ResourceName(nodeResource.XPUCapacityLabel)]; hasGPU {
+ nodeResource.AvailableXPU -= parseQuantityToInt64(requestedGPU)
+ }
+ if memoryRequest, hasMemory := container.Resources.Requests[v1.ResourceMemory]; hasMemory {
+ nodeResource.AvailableMem -= getMem(parseQuantityToInt64(memoryRequest))
+ }
+ if cpuRequest, hasCPU := container.Resources.Requests[v1.ResourceCPU]; hasCPU {
+ nodeResource.AvailableCPU -= millicoresToCores(cpuRequest.MilliValue())
+ }
+ }
+
+ nodeResourcesMap[pod.Spec.NodeName] = nodeResource
+ }
+
+ return nodeResourcesMap, nil
+}
+
+func getXPULabel(node v1.Node, config *config.Config) (string, string) {
+ if _, found := node.Labels[config.Space.GPUModelLabel]; found {
+ //for default cluster
+ return "nvidia.com/gpu", config.Space.GPUModelLabel
+ }
+ if _, found := node.Labels["nvidia.com/nvidia_name"]; found {
+ //for k3s cluster
+ return "nvidia.com/gpu", "nvidia.com/nvidia_name"
+ }
+ if _, found := node.Labels["kubemore_xpu_type"]; found {
+ //for huawei gpu
+ return "huawei.com/Ascend910", "kubemore_xpu_type"
+ }
+ if _, found := node.Labels["huawei.accelerator"]; found {
+ //for huawei gpu
+ return "huawei.com/Ascend910", "huawei.accelerator"
+ }
+ if _, found := node.Labels["accelerator/huawei-npu"]; found {
+ //for huawei gpu
+ return "huawei.com/Ascend910", "accelerator/huawei-npu"
+ }
+ return "", ""
+}
+
+func getMem(memByte int64) float32 {
+ memGB := float32(memByte) / (1024 * 1024 * 1024)
+ return memGB
+}
+
+func millicoresToCores(millicores int64) float64 {
+ cores := float64(millicores) / 1000.0
+ return math.Round(cores*10) / 10
+}
+
+func parseQuantityToInt64(q resource.Quantity) int64 {
+ if q.IsZero() {
+ return 0
+ }
+ value, _ := q.AsInt64()
+ return value
+}
+
+
+
package common
+
+import (
+ "fmt"
+ "strings"
+)
+
+// UniqueSpaceAppName generates a unique app name for space deployment
+func UniqueSpaceAppName(prefix, namespace, name string, spaceID int64) string {
+ encodedSpaceID := numberToString(spaceID)
+ // max length of uniqueAppName is 63
+ avaiMaxLen := 63 - len(encodedSpaceID) - 3
+ nameSpaceAndName := fmt.Sprintf("%s-%s", namespace, name)
+ if len(nameSpaceAndName) > avaiMaxLen {
+ nameSpaceAndName = nameSpaceAndName[:avaiMaxLen]
+ }
+ uniqueAppName := fmt.Sprintf("%s-%s-%s", prefix, nameSpaceAndName, encodedSpaceID)
+ return strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(uniqueAppName, "_", "-"), ".", "-"))
+}
+
+func parseUniqueSpaceAppName(spaceAppName string) (spaceID int64, err error) {
+ nameParts := strings.Split(spaceAppName, "-")
+ spaceIDStr := nameParts[len(nameParts)-1]
+ // decode space id
+ return stringToNumber(spaceIDStr)
+}
+
+// NumberToString encodes a number into a shorter string representation without padding
+func numberToString(num int64) string {
+ alphabet := "0123456789abcdefghijklmnopqrstuvwxyz"
+ var encodedBuilder strings.Builder
+ base := int64(len(alphabet))
+
+ for num > 0 {
+ remainder := num % base
+ num /= base
+ encodedBuilder.WriteByte(alphabet[remainder])
+ }
+
+ // Reverse the encoded string since we've built it backwards
+ encodedStr := encodedBuilder.String()
+ runes := []rune(encodedStr)
+ for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 {
+ runes[i], runes[j] = runes[j], runes[i]
+ }
+
+ return string(runes)
+}
+
+// StringToNumber decodes a string back into the original number without padding
+func stringToNumber(encoded string) (int64, error) {
+ alphabet := "0123456789abcdefghijklmnopqrstuvwxyz"
+ alphabetMap := make(map[rune]int64)
+ for i, c := range alphabet {
+ alphabetMap[c] = int64(i)
+ }
+
+ var num int64
+ base := int64(len(alphabet))
+
+ for _, r := range encoded {
+ num = num*base + alphabetMap[r]
+ }
+
+ return num, nil
+}
+
+
+
package common
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+)
+
+func JsonStrToMap(jsonStr string) (map[string]string, error) {
+ var resMap map[string]string
+ if len(strings.Trim(jsonStr, " ")) == 0 {
+ return map[string]string{}, nil
+ }
+ err := json.Unmarshal([]byte(jsonStr), &resMap)
+ return resMap, err
+}
+
+func GetNamespaceAndNameFromGitPath(gitpath string) (string, string, error) {
+ if gitpath == "" {
+ return "", "", fmt.Errorf("empty git path %s", gitpath)
+ }
+ var fields []string
+ idx := strings.Index(gitpath, "_")
+ if idx > -1 && idx+1 < len(gitpath) {
+ fields = strings.Split(gitpath[idx+1:], "/")
+ if len(fields) != 2 {
+ return "", "", fmt.Errorf("empty git path %s", gitpath)
+ }
+ } else {
+ return "", "", fmt.Errorf("empty git path %s", gitpath)
+ }
+ return fields[0], fields[1], nil
+}
+
+
+
package deploy
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/bwmarrin/snowflake"
+ "github.com/google/uuid"
+ "github.com/nats-io/nats.go/jetstream"
+ "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/builder/deploy/imagebuilder"
+ "opencsg.com/csghub-server/builder/deploy/imagerunner"
+ "opencsg.com/csghub-server/builder/deploy/scheduler"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type Deployer interface {
+ Deploy(ctx context.Context, dr types.DeployRepo) (deployID int64, err error)
+ Status(ctx context.Context, dr types.DeployRepo, needDetails bool) (srvName string, status int, instances []types.Instance, err error)
+ Logs(ctx context.Context, dr types.DeployRepo) (*MultiLogReader, error)
+ Stop(ctx context.Context, dr types.DeployRepo) (err error)
+ Purge(ctx context.Context, dr types.DeployRepo) (err error)
+ Wakeup(ctx context.Context, dr types.DeployRepo) (err error)
+ Exist(ctx context.Context, dr types.DeployRepo) (bool, error)
+ GetReplica(ctx context.Context, dr types.DeployRepo) (int, int, []types.Instance, error)
+ InstanceLogs(ctx context.Context, dr types.DeployRepo) (*MultiLogReader, error)
+ ListCluster(ctx context.Context) ([]types.ClusterRes, error)
+ GetClusterById(ctx context.Context, clusterId string) (*types.ClusterRes, error)
+ UpdateCluster(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error)
+ UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq, deploy *database.Deploy) error
+ StartDeploy(ctx context.Context, deploy *database.Deploy) error
+ CheckResourceAvailable(ctx context.Context, clusterId string, orderDetailID int64, hardWare *types.HardWare) (bool, error)
+ SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error)
+ ListEvaluations(context.Context, string, int, int) (*types.ArgoWorkFlowListRes, error)
+ DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error
+ GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error)
+}
+
+var _ Deployer = (*deployer)(nil)
+
+type deployer struct {
+ s scheduler.Scheduler
+ ib imagebuilder.Builder
+ ir imagerunner.Runner
+
+ store database.DeployTaskStore
+ spaceStore database.SpaceStore
+ spaceResourceStore database.SpaceResourceStore
+ runnerStatuscache map[string]types.StatusResponse
+ internalRootDomain string
+ sfNode *snowflake.Node
+ eventPub *event.EventPublisher
+ rtfm database.RuntimeFrameworksStore
+ userResStore database.UserResourcesStore
+ c DeployConfig
+ us database.UserStore
+}
+
+func newDeployer(s scheduler.Scheduler, ib imagebuilder.Builder, ir imagerunner.Runner, c DeployConfig) (*deployer, error) {
+ store := database.NewDeployTaskStore()
+ node, err := snowflake.NewNode(1)
+ if err != nil || node == nil {
+ slog.Error("fail to generate uuid for inference service name", slog.Any("error", err))
+ return nil, err
+ }
+ d := &deployer{
+ s: s,
+ ib: ib,
+ ir: ir,
+ store: store,
+ spaceStore: database.NewSpaceStore(),
+ spaceResourceStore: database.NewSpaceResourceStore(),
+ runnerStatuscache: make(map[string]types.StatusResponse),
+ sfNode: node,
+ eventPub: &event.DefaultEventPublisher,
+ rtfm: database.NewRuntimeFrameworksStore(),
+ userResStore: database.NewUserResourcesStore(),
+ c: c,
+ us: database.NewUserStore(),
+ }
+
+ go d.refreshStatus()
+ if d.c.IsMasterHost {
+ go func() {
+ err = d.s.Run()
+ if err != nil {
+ slog.Error("run scheduler failed", slog.Any("error", err))
+ }
+ }()
+ go d.startAccounting()
+ }
+ return d, nil
+}
+
+func (d *deployer) GenerateUniqueSvcName(dr types.DeployRepo) string {
+ uniqueSvcName := ""
+ if dr.Type == types.SpaceType {
+ // space
+ fields := strings.Split(dr.Path, "/")
+ uniqueSvcName = common.UniqueSpaceAppName("u", fields[0], fields[1], dr.SpaceID)
+ } else if dr.Type == types.ServerlessType {
+ // model serverless
+ fields := strings.Split(dr.Path, "/")
+ uniqueSvcName = common.UniqueSpaceAppName("s", fields[0], fields[1], dr.RepoID)
+ } else {
+ // model inference
+ // generate unique service name from uuid when create new deploy by snowflake
+ uniqueSvcName = d.sfNode.Generate().Base36()
+ }
+ return uniqueSvcName
+}
+
+func (d *deployer) serverlessDeploy(ctx context.Context, dr types.DeployRepo) (*database.Deploy, error) {
+ var (
+ deploy *database.Deploy
+ err error
+ )
+ slog.Info("do deployer.serverlessDeploy check type", slog.Any("dr.Type", dr.Type))
+ if dr.Type == types.SpaceType {
+ deploy, err = d.store.GetLatestDeployBySpaceID(ctx, dr.SpaceID)
+ } else {
+ deploy, err = d.store.GetServerlessDeployByRepID(ctx, dr.RepoID)
+ }
+ if err == sql.ErrNoRows {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("fail to find space or serverless deploy, spaceid:%v, repoid:%v, %w", dr.SpaceID, dr.RepoID, err)
+ }
+ if deploy == nil {
+ return nil, nil
+ }
+ deploy.UserUUID = dr.UserUUID
+ deploy.SKU = dr.SKU
+ // dr.ImageID is not null for nginx space, otherwise it's ""
+ deploy.ImageID = dr.ImageID
+ deploy.Annotation = dr.Annotation
+ deploy.Env = dr.Env
+ deploy.Hardware = dr.Hardware
+ deploy.RuntimeFramework = dr.RuntimeFramework
+ deploy.Secret = dr.Secret
+ deploy.SecureLevel = dr.SecureLevel
+ deploy.ContainerPort = dr.ContainerPort
+ deploy.Template = dr.Template
+ deploy.MinReplica = dr.MinReplica
+ deploy.MaxReplica = dr.MaxReplica
+ slog.Debug("do deployer.serverlessDeploy", slog.Any("dr", dr), slog.Any("deploy", deploy))
+ err = d.store.UpdateDeploy(ctx, deploy)
+ if err != nil {
+ return nil, fmt.Errorf("fail reset deploy image, %w", err)
+ }
+ slog.Debug("return deployer.serverlessDeploy", slog.Any("dr", dr), slog.Any("deploy", deploy))
+ return deploy, nil
+}
+
+func (d *deployer) dedicatedDeploy(ctx context.Context, dr types.DeployRepo) (*database.Deploy, error) {
+ uniqueSvcName := d.GenerateUniqueSvcName(dr)
+ if len(uniqueSvcName) < 1 {
+ return nil, fmt.Errorf("fail to generate uuid for deploy")
+ }
+ deploy := &database.Deploy{
+ DeployName: dr.DeployName,
+ SpaceID: dr.SpaceID,
+ GitPath: dr.Path,
+ GitBranch: dr.GitBranch,
+ Secret: dr.Secret,
+ Template: dr.Template,
+ Env: dr.Env,
+ Hardware: dr.Hardware,
+ ImageID: dr.ImageID,
+ ModelID: dr.ModelID,
+ UserID: dr.UserID,
+ RepoID: dr.RepoID,
+ RuntimeFramework: dr.RuntimeFramework,
+ ContainerPort: dr.ContainerPort,
+ Annotation: dr.Annotation,
+ MinReplica: dr.MinReplica,
+ MaxReplica: dr.MaxReplica,
+ ClusterID: dr.ClusterID,
+ SecureLevel: dr.SecureLevel,
+ SvcName: uniqueSvcName,
+ Type: dr.Type,
+ UserUUID: dr.UserUUID,
+ SKU: dr.SKU,
+ OrderDetailID: dr.OrderDetailID,
+ }
+ err := d.store.CreateDeploy(ctx, deploy)
+ return deploy, err
+}
+
+func (d *deployer) buildDeploy(ctx context.Context, dr types.DeployRepo) (*database.Deploy, error) {
+ var deploy *database.Deploy = nil
+ var err error = nil
+ slog.Info("do deployer.buildDeploy check type", slog.Any("dr.Type", dr.Type))
+ if dr.Type == types.SpaceType || dr.Type == types.ServerlessType {
+ // space case: SpaceID>0 and ModelID=0, reuse latest deploy of spaces
+ deploy, err = d.serverlessDeploy(ctx, dr)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check serverless deploy for spaceID %v, %w", dr.SpaceID, err)
+ }
+ }
+ slog.Info("do deployer.buildDeploy", slog.Any("dr", dr), slog.Any("deploy", deploy))
+ if deploy == nil {
+ // create new deploy for model inference and no latest deploy of space
+ deploy, err = d.dedicatedDeploy(ctx, dr)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ slog.Info("return deployer.buildDeploy", slog.Any("dr", dr), slog.Any("deploy", deploy))
+ return deploy, nil
+}
+
+func (d *deployer) Deploy(ctx context.Context, dr types.DeployRepo) (int64, error) {
+ //check reserved resource
+ if dr.OrderDetailID != 0 {
+ ur, err := d.userResStore.FindUserResourcesByOrderDetailId(ctx, dr.UserUUID, dr.OrderDetailID)
+ if err != nil {
+ return -1, fmt.Errorf("failed to check reserved resource, %w", err)
+ }
+ if ur.DeployId != 0 {
+ return -1, fmt.Errorf("the reserved resource is already used by deploy id %v", ur.DeployId)
+ }
+ }
+ deploy, err := d.buildDeploy(ctx, dr)
+ slog.Info("do deployer.Deploy", slog.Any("dr", dr), slog.Any("deploy", deploy))
+ if err != nil || deploy == nil {
+ return -1, fmt.Errorf("failed to create deploy in db, %w", err)
+ }
+ // skip build step for model as inference
+ bldTaskStatus := 0
+ bldTaskMsg := ""
+
+ imgStrLen := len(strings.Trim(deploy.ImageID, " "))
+ slog.Info("do deployer.Deploy check image", slog.Any("deploy.ImageID", deploy.ImageID), slog.Any("imgStrLen", imgStrLen))
+ if imgStrLen > 0 {
+ bldTaskStatus = scheduler.BuildSkip
+ bldTaskMsg = "Skip"
+ }
+ slog.Info("create build task", slog.Any("bldTaskStatus", bldTaskStatus), slog.Any("bldTaskMsg", bldTaskMsg))
+ buildTask := &database.DeployTask{
+ DeployID: deploy.ID,
+ TaskType: 0,
+ Status: bldTaskStatus,
+ Message: bldTaskMsg,
+ }
+ err = d.store.CreateDeployTask(ctx, buildTask)
+ if err != nil {
+ return -1, fmt.Errorf("create deploy task failed: %w", err)
+ }
+ runTask := &database.DeployTask{
+ DeployID: deploy.ID,
+ TaskType: 1,
+ }
+ err = d.store.CreateDeployTask(ctx, runTask)
+ if err != nil {
+ return -1, fmt.Errorf("create deploy task failed: %w", err)
+ }
+
+ go func() { _ = d.s.Queue(buildTask.ID) }()
+
+ return deploy.ID, nil
+}
+
+func (d *deployer) refreshStatus() {
+ for {
+ ctxTimeout, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ status, err := d.ir.StatusAll(ctxTimeout)
+ cancel()
+ if err != nil {
+ slog.Error("refresh status all failed", slog.Any("error", err))
+ } else {
+ slog.Debug("status all cached", slog.Any("status", d.runnerStatuscache))
+ d.runnerStatuscache = status
+ }
+
+ time.Sleep(5 * time.Second)
+ }
+}
+
+func (d *deployer) Status(ctx context.Context, dr types.DeployRepo, needDetails bool) (string, int, []types.Instance, error) {
+ deploy, err := d.store.GetDeployByID(ctx, dr.DeployID)
+ if err != nil || deploy == nil {
+ slog.Error("fail to get deploy by deploy id", slog.Any("DeployID", deploy.ID), slog.Any("error", err))
+ return "", common.Stopped, nil, fmt.Errorf("can't get deploy, %w", err)
+ }
+ svcName := deploy.SvcName
+ // srvName := common.UniqueSpaceAppName(dr.Namespace, dr.Name, dr.SpaceID)
+ rstatus, found := d.runnerStatuscache[svcName]
+ if !found {
+ slog.Debug("status cache miss", slog.String("svc_name", svcName))
+ if deploy.Status == common.Running {
+ // service was Stopped or delete, so no running instance
+ return svcName, common.Stopped, nil, nil
+ }
+ return svcName, deploy.Status, nil, nil
+ }
+ deployStatus := rstatus.Code
+ if dr.ModelID > 0 {
+ targetID := dr.DeployID // support model deploy with multi-instance
+ status, err := d.ir.Status(ctx, &types.StatusRequest{
+ ClusterID: dr.ClusterID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ SvcName: deploy.SvcName,
+ ID: targetID,
+ NeedDetails: needDetails,
+ })
+ if err != nil {
+ slog.Error("fail to get status by deploy id", slog.Any("DeployID", deploy.ID), slog.Any("error", err))
+ return "", common.RunTimeError, nil, fmt.Errorf("can't get deploy status, %w", err)
+ }
+ rstatus.Instances = status.Instances
+ deployStatus = status.Code
+
+ }
+ if rstatus.DeployID == 0 || rstatus.DeployID >= deploy.ID {
+ return svcName, deployStatus, rstatus.Instances, nil
+ }
+ return svcName, deployStatus, rstatus.Instances, nil
+}
+
+func (d *deployer) Logs(ctx context.Context, dr types.DeployRepo) (*MultiLogReader, error) {
+ // get latest Deploy
+ deploy, err := d.store.GetLatestDeployBySpaceID(ctx, dr.SpaceID)
+ if err != nil {
+ return nil, fmt.Errorf("can't get space delopyment,%w", err)
+ }
+
+ slog.Debug("get logs for space", slog.Any("deploy", deploy), slog.Int64("space_id", dr.SpaceID))
+ buildLog, err := d.ib.Logs(ctx, &imagebuilder.LogsRequest{
+ OrgName: dr.Namespace,
+ SpaceName: dr.Name,
+ BuildID: strconv.FormatInt(deploy.ID, 10),
+ })
+ if err != nil {
+ // return nil, fmt.Errorf("connect to imagebuilder failed: %w", err)
+ slog.Error("failed to read log from image builder", slog.Any("error", err))
+ }
+
+ targetID := dr.SpaceID // support space only one instance
+ if dr.SpaceID == 0 {
+ targetID = dr.DeployID // support model deploy with multi-instance
+ }
+ runLog, err := d.ir.Logs(ctx, &types.LogsRequest{
+ ID: targetID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ SvcName: deploy.SvcName,
+ ClusterID: dr.ClusterID,
+ })
+ if err != nil {
+ slog.Error("failed to read log from image runner", slog.Any("error", err))
+ // return nil, fmt.Errorf("connect to imagerunner failed: %w", err)
+ }
+
+ return NewMultiLogReader(buildLog, runLog), nil
+}
+
+func (d *deployer) Stop(ctx context.Context, dr types.DeployRepo) error {
+ targetID := dr.SpaceID // support space only one instance
+ if dr.SpaceID == 0 {
+ targetID = dr.DeployID // support model deploy with multi-instance
+ }
+ resp, err := d.ir.Stop(ctx, &types.StopRequest{
+ ID: targetID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ SvcName: dr.SvcName,
+ ClusterID: dr.ClusterID,
+ })
+ if err != nil {
+ slog.Error("deployer stop deploy", slog.Any("runner_resp", resp), slog.Int64("space_id", dr.SpaceID), slog.Any("deploy_id", dr.DeployID), slog.Any("error", err))
+ }
+ // release resource if it's a order case
+ if dr.OrderDetailID != 0 {
+ ur, err := d.userResStore.FindUserResourcesByOrderDetailId(ctx, dr.UserUUID, dr.OrderDetailID)
+ if err != nil {
+ return fmt.Errorf("fail to find user resource, %w", err)
+ }
+ ur.DeployId = 0
+ err = d.userResStore.UpdateDeployId(ctx, ur)
+ if err != nil {
+ return fmt.Errorf("fail to release resource, %w", err)
+ }
+ }
+ return err
+}
+
+func (d *deployer) Purge(ctx context.Context, dr types.DeployRepo) error {
+ targetID := dr.SpaceID // support space only one instance
+ if dr.SpaceID == 0 {
+ targetID = dr.DeployID // support model deploy with multi-instance
+ }
+ resp, err := d.ir.Purge(ctx, &types.PurgeRequest{
+ ID: targetID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ SvcName: dr.SvcName,
+ ClusterID: dr.ClusterID,
+ DeployType: dr.Type,
+ UserID: dr.UserUUID,
+ })
+ if err != nil {
+ slog.Error("deployer stop deploy", slog.Any("runner_resp", resp), slog.Int64("space_id", dr.SpaceID), slog.Any("deploy_id", dr.DeployID), slog.Any("error", err))
+ }
+ return err
+}
+
+func (d *deployer) Wakeup(ctx context.Context, dr types.DeployRepo) error {
+ // srvName := common.UniqueSpaceAppName(dr.Namespace, dr.Name, dr.SpaceID)
+ svcName := dr.SvcName
+ srvURL := fmt.Sprintf("http://%s.%s", svcName, d.internalRootDomain)
+ // Create a new HTTP client with a timeout
+ client := &http.Client{
+ Timeout: 10 * time.Second,
+ }
+
+ // Send a GET request to wake up the service
+ resp, err := client.Get(srvURL)
+ if err != nil {
+ fmt.Printf("Error sending request to Knative service: %s\n", err)
+ return fmt.Errorf("failed call service endpoint, %w", err)
+ }
+ defer resp.Body.Close()
+
+ // Check if the request was successful
+ if resp.StatusCode == http.StatusOK {
+ return nil
+ } else {
+ return fmt.Errorf("space endpoint status not ok, status:%d", resp.StatusCode)
+ }
+}
+
+func (d *deployer) Exist(ctx context.Context, dr types.DeployRepo) (bool, error) {
+ targetID := dr.SpaceID // support space only one instance
+ if dr.SpaceID == 0 {
+ targetID = dr.DeployID // support model deploy with multi-instance
+ }
+ req := &types.CheckRequest{
+ ID: targetID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ SvcName: dr.SvcName,
+ ClusterID: dr.ClusterID,
+ }
+ resp, err := d.ir.Exist(ctx, req)
+ if err != nil {
+ slog.Error("fail to check deploy", slog.Any("req", req), slog.Any("error", err))
+ return true, err
+ }
+
+ if resp.Code == -1 {
+ // service check with error
+ slog.Error("deploy check result", slog.Any("resp", resp))
+ return true, errors.New("fail to check deploy instance")
+ } else if resp.Code == 1 {
+ // service exist
+ return true, nil
+ }
+ // service not exist
+ return false, nil
+}
+
+func (d *deployer) GetReplica(ctx context.Context, dr types.DeployRepo) (int, int, []types.Instance, error) {
+ targetID := dr.SpaceID // support space only one instance
+ if dr.SpaceID == 0 {
+ targetID = dr.DeployID // support model deploy with multi-instance
+ }
+ req := &types.StatusRequest{
+ ID: targetID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ ClusterID: dr.ClusterID,
+ SvcName: dr.SvcName,
+ }
+ resp, err := d.ir.GetReplica(ctx, req)
+ if err != nil {
+ slog.Warn("fail to get deploy replica with error", slog.Any("req", req), slog.Any("error", err))
+ return 0, 0, []types.Instance{}, err
+ }
+ return resp.ActualReplica, resp.DesiredReplica, resp.Instances, nil
+}
+
+func (d *deployer) InstanceLogs(ctx context.Context, dr types.DeployRepo) (*MultiLogReader, error) {
+ slog.Debug("get logs for deploy", slog.Any("deploy", dr))
+
+ targetID := dr.SpaceID // support space only one instance
+ if dr.SpaceID == 0 {
+ targetID = dr.DeployID // support model deploy with multi-instance
+ }
+ runLog, err := d.ir.InstanceLogs(ctx, &types.InstanceLogsRequest{
+ ID: targetID,
+ OrgName: dr.Namespace,
+ RepoName: dr.Name,
+ ClusterID: dr.ClusterID,
+ SvcName: dr.SvcName,
+ InstanceName: dr.InstanceName,
+ })
+ if err != nil {
+ slog.Error("failed to read log from deploy runner", slog.Any("error", err))
+ // return nil, fmt.Errorf("connect to imagerunner failed: %w", err)
+ }
+
+ return NewMultiLogReader(nil, runLog), nil
+}
+
+func (d *deployer) ListCluster(ctx context.Context) ([]types.ClusterRes, error) {
+ resp, err := d.ir.ListCluster(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var result []types.ClusterRes
+ for _, c := range resp {
+ resources := make([]types.NodeResourceInfo, 0)
+ for _, node := range c.Nodes {
+ resources = append(resources, node)
+ }
+ result = append(result, types.ClusterRes{
+ ClusterID: c.ClusterID,
+ Region: c.Region,
+ Zone: c.Zone,
+ Provider: c.Provider,
+ Resources: resources,
+ })
+ }
+ return result, err
+}
+
+func (d *deployer) GetClusterById(ctx context.Context, clusterId string) (*types.ClusterRes, error) {
+ resp, err := d.ir.GetClusterById(ctx, clusterId)
+ if err != nil {
+ return nil, err
+ }
+ // get reserved resources
+ userResources, err := d.userResStore.GetReservedUserResources(ctx, "", clusterId)
+ if err != nil {
+ return nil, err
+ }
+ resources := make([]types.NodeResourceInfo, 0)
+ for _, node := range resp.Nodes {
+ resources = append(resources, node)
+ }
+ for _, r := range userResources {
+ for i := range resources {
+ if resources[i].AvailableXPU >= int64(r.XPUNum) {
+ resources[i].AvailableXPU -= int64(r.XPUNum)
+ resources[i].ReservedXPU += int64(r.XPUNum)
+ }
+ }
+ }
+ result := types.ClusterRes{
+ ClusterID: resp.ClusterID,
+ Region: resp.Region,
+ Zone: resp.Zone,
+ Provider: resp.Provider,
+ Resources: resources,
+ }
+ return &result, err
+}
+
+func (d *deployer) UpdateCluster(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error) {
+ resp, err := d.ir.UpdateCluster(ctx, &data)
+ return (*types.UpdateClusterResponse)(resp), err
+}
+
+// UpdateDeploy implements Deployer.
+func (d *deployer) UpdateDeploy(ctx context.Context, dur *types.DeployUpdateReq, deploy *database.Deploy) error {
+ var (
+ frame *database.RuntimeFramework
+ resource *database.SpaceResource
+ hardware *types.HardWare
+ err error
+ )
+
+ if dur.RuntimeFrameworkID != nil {
+ frame, err = d.rtfm.FindEnabledByID(ctx, *dur.RuntimeFrameworkID)
+ if err != nil || frame == nil {
+ return fmt.Errorf("can't find available runtime framework %v, %w", *dur.RuntimeFrameworkID, err)
+ }
+ }
+
+ if dur.ResourceID != nil {
+ resource, err = d.spaceResourceStore.FindByID(ctx, *dur.ResourceID)
+ if err != nil {
+ return fmt.Errorf("error finding space resource %d, %w", *dur.ResourceID, err)
+ }
+ err = json.Unmarshal([]byte(resource.Resources), &hardware)
+ if err != nil {
+ return fmt.Errorf("invalid resource hardware setting, %w", err)
+ }
+ deploy.Hardware = resource.Resources
+ deploy.SKU = strconv.FormatInt(resource.ID, 10)
+ } else {
+ err = json.Unmarshal([]byte(deploy.Hardware), &hardware)
+ if err != nil {
+ return fmt.Errorf("invalid deploy hardware setting, %w", err)
+ }
+ }
+
+ if dur.DeployName != nil {
+ deploy.DeployName = *dur.DeployName
+ }
+ if dur.Env != nil {
+ deploy.Env = *dur.Env
+ }
+
+ if frame != nil {
+ // choose image
+ containerImg := frame.FrameCpuImage
+ if hardware != nil {
+ // use gpu image
+ if hardware.Gpu.Num != "" {
+ containerImg = frame.FrameImage
+ } else if hardware.Npu.Num != "" {
+ containerImg = frame.FrameNpuImage
+ }
+ }
+ deploy.ImageID = containerImg
+ deploy.RuntimeFramework = frame.FrameName
+ deploy.ContainerPort = frame.ContainerPort
+ }
+
+ if dur.MinReplica != nil {
+ deploy.MinReplica = *dur.MinReplica
+ }
+
+ if dur.MaxReplica != nil {
+ deploy.MaxReplica = *dur.MaxReplica
+ }
+
+ if deploy.MaxReplica < deploy.MinReplica {
+ return fmt.Errorf("invalid min/max replica %d/%d", deploy.MinReplica, deploy.MaxReplica)
+ }
+
+ if dur.Revision != nil {
+ deploy.GitBranch = *dur.Revision
+ }
+
+ if dur.SecureLevel != nil {
+ deploy.SecureLevel = *dur.SecureLevel
+ }
+ if dur.ClusterID != nil {
+ deploy.ClusterID = *dur.ClusterID
+ }
+
+ // update deploy table
+ err = d.store.UpdateDeploy(ctx, deploy)
+ if err != nil {
+ return fmt.Errorf("failed to update deploy, %w", err)
+ }
+
+ return nil
+}
+
+func (d *deployer) StartDeploy(ctx context.Context, deploy *database.Deploy) error {
+ deploy.Status = common.Pending
+ // update deploy table
+ err := d.store.UpdateDeploy(ctx, deploy)
+ if err != nil {
+ return fmt.Errorf("failed to update deploy, %w", err)
+ }
+
+ // start model as inference/serverless task
+ runTask := &database.DeployTask{
+ DeployID: deploy.ID,
+ TaskType: 1,
+ }
+ err = d.store.CreateDeployTask(ctx, runTask)
+ if err != nil {
+ return fmt.Errorf("create deploy task failed: %w", err)
+ }
+
+ go func() { _ = d.s.Queue(runTask.ID) }()
+
+ // update resource if it's a order case
+ if deploy.OrderDetailID != 0 {
+ ur, err := d.userResStore.FindUserResourcesByOrderDetailId(ctx, deploy.UserUUID, deploy.OrderDetailID)
+ if err != nil {
+ return fmt.Errorf("fail to find user resource, %w", err)
+ }
+ ur.DeployId = deploy.ID
+ err = d.userResStore.UpdateDeployId(ctx, ur)
+ if err != nil {
+ return fmt.Errorf("fail to update user resource, %w", err)
+ }
+ }
+
+ return nil
+}
+
+// accounting timer
+func (d *deployer) startAccounting() {
+ if d.c.ChargingEnable {
+ d.startAcctBalanceConsuming()
+ d.startAcctOrderConsuming()
+ }
+ d.startAcctFeeing()
+}
+
+func (d *deployer) startAcctBalanceConsuming() {
+ for {
+ consumer, err := d.eventPub.CreateNoBalanceConsumer()
+ if err != nil {
+ slog.Error("fail to create continuous polling balance consumer", slog.Any("error", err))
+ } else {
+ _, err = consumer.Consume(d.acctBalanceConsumerCallback)
+ if err != nil {
+ slog.Error("fail to begin consuming balance message", slog.Any("error", err))
+ } else {
+ break
+ }
+ }
+ time.Sleep(1 * time.Second)
+ }
+}
+
+func (d *deployer) startAcctOrderConsuming() {
+ for {
+ consumer, err := d.eventPub.CreateOrderExporedConsumer()
+ if err != nil {
+ slog.Error("fail to create continuous polling order expired consumer", slog.Any("error", err))
+ } else {
+ _, err = consumer.Consume(d.acctOrderExpiredConsumerCallback)
+ if err != nil {
+ slog.Error("fail to begin consuming order expired message", slog.Any("error", err))
+ } else {
+ break
+ }
+ }
+ time.Sleep(1 * time.Second)
+ }
+}
+
+func (d *deployer) getResourceMap() map[string]string {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ resList, err := d.spaceResourceStore.FindAll(ctx)
+ resources := make(map[string]string)
+ if err != nil {
+ slog.Error("failed to get hub resource", slog.Any("error", err))
+ } else {
+ for _, res := range resList {
+ resources[strconv.FormatInt(res.ID, 10)] = res.Name
+ }
+ }
+ return resources
+}
+
+func (d *deployer) startAcctFeeing() {
+ for {
+ resMap := d.getResourceMap()
+ slog.Debug("get resources map", slog.Any("resMap", resMap))
+ for _, svc := range d.runnerStatuscache {
+ d.startAcctRequestFee(resMap, svc)
+ }
+ // accounting interval in min, get from env config
+ time.Sleep(time.Duration(d.eventPub.SyncInterval) * time.Minute)
+ }
+}
+
+func (d *deployer) acctBalanceConsumerCallback(msg jetstream.Msg) {
+ msgStr := string(msg.Data())
+ slog.Debug("Received an insufficient balance message", slog.Any("msg", msgStr))
+ if d.c.AutoCleanInstance {
+ err := d.cleanUserSvcForInsufficientBalanceWithRetry(msg.Data())
+ if err != nil {
+ slog.Warn("fail to clean user service for insufficient balance with retry 3 times", slog.Any("msg", msgStr), slog.Any("err", err))
+ }
+ }
+ err := msg.Ack()
+ if err != nil {
+ slog.Warn("fail to ack after processing insufficient balance message", slog.Any("msg", msgStr), slog.Any("err", err))
+ }
+}
+
+func (d *deployer) acctOrderExpiredConsumerCallback(msg jetstream.Msg) {
+ slog.Debug("Received an order expired message", slog.Any("msg", string(msg.Data())))
+ err := msg.Ack()
+ if err != nil {
+ slog.Warn("fail to ack after processing order expired message", slog.Any("msg", string(msg.Data())))
+ }
+}
+
+func (d *deployer) startAcctRequestFee(resMap map[string]string, svcRes types.StatusResponse) {
+ // ignore not ready svc
+ if svcRes.Code != common.Running {
+ return
+ }
+ // ignore deploy without sku resource
+ if len(svcRes.DeploySku) < 1 {
+ return
+ }
+ resName, exists := resMap[svcRes.DeploySku]
+ if !exists {
+ slog.Warn("Did not find space resource by id for metering", slog.Any("id", svcRes.DeploySku), slog.Any("deploy_id", svcRes.DeployID), slog.Any("svc_name", svcRes.ServiceName))
+ return
+ }
+ slog.Debug("metering service", slog.Any("svcRes", svcRes))
+ sceneType := getValidSceneType(svcRes.DeployType)
+ if sceneType == types.SceneUnknow {
+ slog.Error("invalid deploy type of service for metering", slog.Any("svcRes", svcRes))
+ return
+ }
+ extra := ""
+ if svcRes.OrderDetailID != 0 {
+ extra = fmt.Sprintf("{ \"order_detail_id\": \"%d\" }", svcRes.OrderDetailID)
+ }
+ event := types.METERING_EVENT{
+ Uuid: uuid.New(),
+ UserUUID: svcRes.UserID,
+ Value: int64(d.eventPub.SyncInterval),
+ ValueType: types.TimeDurationMinType,
+ Scene: int(sceneType),
+ OpUID: "",
+ ResourceID: svcRes.DeploySku,
+ ResourceName: resName,
+ CustomerID: svcRes.ServiceName,
+ CreatedAt: time.Now(),
+ Extra: extra,
+ }
+ str, err := json.Marshal(event)
+ if err != nil {
+ slog.Error("error marshal metering event", slog.Any("event", event), slog.Any("error", err))
+ return
+ }
+ err = d.eventPub.PublishMeteringEvent(str)
+ if err != nil {
+ slog.Error("failed to pub metering event", slog.Any("data", string(str)), slog.Any("error", err))
+ } else {
+ slog.Debug("pub metering event success", slog.Any("data", string(str)))
+ }
+}
+
+func getValidSceneType(deployType int) types.SceneType {
+ switch deployType {
+ case types.SpaceType:
+ return types.SceneSpace
+ case types.InferenceType:
+ return types.SceneModelInference
+ case types.FinetuneType:
+ return types.SceneModelFinetune
+ case types.ServerlessType:
+ return types.SceneModelInference
+ default:
+ return types.SceneUnknow
+ }
+}
+
+func (d *deployer) CheckResourceAvailable(ctx context.Context, clusterId string, orderDetailID int64, hardWare *types.HardWare) (bool, error) {
+ // backward compatibility for old api
+ if clusterId == "" {
+ clusters, err := d.ListCluster(ctx)
+ if err != nil {
+ return false, err
+ }
+ if len(clusters) == 0 {
+ return false, fmt.Errorf("can not list clusters")
+ }
+ clusterId = clusters[0].ClusterID
+ }
+ clusterResources, err := d.GetClusterById(ctx, clusterId)
+ if err != nil {
+ return false, err
+ }
+ if orderDetailID != 0 {
+ ur, err := d.userResStore.FindUserResourcesByOrderDetailId(ctx, "", orderDetailID)
+ if err != nil {
+ return false, fmt.Errorf("failed to check user resources by order detail id %d: %v", orderDetailID, err)
+ }
+ if ur.DeployId != 0 {
+ return false, fmt.Errorf("order detail id %d is already used", orderDetailID)
+ }
+ return true, nil
+ }
+ if !CheckResource(clusterResources, hardWare) {
+ return false, fmt.Errorf("required resource is not enough")
+ }
+
+ return true, nil
+}
+
+func CheckResource(clusterResources *types.ClusterRes, hardware *types.HardWare) bool {
+ mem, err := strconv.Atoi(strings.Replace(hardware.Memory, "Gi", "", -1))
+ if err != nil {
+ slog.Error("failed to parse hardware memory ", slog.Any("error", err))
+ return false
+ }
+ for _, node := range clusterResources.Resources {
+ if float32(mem) <= node.AvailableMem {
+ if hardware.Gpu.Num != "" {
+ gpu, err := strconv.Atoi(hardware.Gpu.Num)
+ if err != nil {
+ slog.Error("failed to parse hardware gpu ", slog.Any("error", err))
+ return false
+ }
+ cpu, err := strconv.Atoi(hardware.Cpu.Num)
+ if err != nil {
+ slog.Error("failed to parse hardware cpu ", slog.Any("error", err))
+ return false
+
+ }
+ if gpu <= int(node.AvailableXPU) && hardware.Gpu.Type == node.XPUModel && cpu <= int(node.AvailableCPU) {
+ return true
+ }
+ } else if hardware.Npu.Num != "" {
+ npu, err := strconv.Atoi(hardware.Npu.Num)
+ if err != nil {
+ slog.Error("failed to parse hardware gpu ", slog.Any("error", err))
+ return false
+ }
+ cpu, err := strconv.Atoi(hardware.Cpu.Num)
+ if err != nil {
+ slog.Error("failed to parse hardware cpu ", slog.Any("error", err))
+ return false
+
+ }
+ if npu <= int(node.AvailableXPU) && hardware.Npu.Type == node.XPUModel && cpu <= int(node.AvailableCPU) {
+ return true
+ }
+ } else {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func (d *deployer) cleanUserSvcForInsufficientBalanceWithRetry(data []byte) error {
+ var err error
+ for i := 0; i < 3; i++ {
+ err = d.cleanUserSvcForInsufficientBalance(data)
+ if err == nil {
+ return nil
+ }
+ }
+ return err
+}
+
+func (d *deployer) cleanUserSvcForInsufficientBalance(data []byte) error {
+ notifyEvt := types.AcctNotify{}
+ err := json.Unmarshal(data, ¬ifyEvt)
+ if err != nil {
+ return fmt.Errorf("failed to unmarshal notify insufficient balance event: %w", err)
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ user, err := d.us.FindByUUID(ctx, notifyEvt.UserUUID)
+ if err != nil {
+ return fmt.Errorf("failed to find user by uuid %s for insufficient balance", notifyEvt.UserUUID)
+ }
+ deploys, err := d.store.GetRunningInferenceAndFinetuneByUserID(ctx, user.ID)
+ if err != nil {
+ return fmt.Errorf("failed to get running inference and finetune by user id %d for insufficient balance, %w", user.ID, err)
+ }
+ for _, deploy := range deploys {
+ if deploy.OrderDetailID > 0 {
+ ur, err := d.userResStore.FindUserResourcesByOrderDetailId(ctx, deploy.UserUUID, deploy.OrderDetailID)
+ if err != nil {
+ slog.Warn("get reserved deploy for insufficient balance", slog.Any("deploy ID", deploy.ID), slog.Any("OrderDetailID", deploy.OrderDetailID), slog.Any("error", err))
+ continue
+ }
+ if ur.StartTime.Before(time.Now()) && ur.EndTime.After(time.Now()) {
+ // should not stop for valid deploy of payment by monthly or yearly
+ continue
+ }
+ }
+ namespace, name, err := common.GetNamespaceAndNameFromGitPath(deploy.GitPath)
+ if err != nil {
+ slog.Warn("get namespace/name for insufficient balance", slog.Any("error", err), slog.Any("GitPath", deploy.GitPath), slog.Any("notifyEvt", notifyEvt))
+ }
+ // delete service
+ deployRepo := types.DeployRepo{
+ DeployID: deploy.ID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: namespace,
+ Name: name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ OrderDetailID: deploy.OrderDetailID,
+ UserUUID: notifyEvt.UserUUID,
+ }
+ slog.Debug("do stop action or insufficient balance", slog.Any("deployRepo", deployRepo))
+ err = d.Stop(ctx, deployRepo)
+ if err != nil {
+ // fail to stop deploy instance, maybe service is gone
+ slog.Warn("stop deploy instance for insufficient balance", slog.Any("error", err), slog.Any("deployRepo", deployRepo), slog.Any("notifyEvt", notifyEvt))
+ }
+
+ exist, err := d.Exist(ctx, deployRepo)
+ if err != nil {
+ slog.Warn("check if deploy instance exists for insufficient balance", slog.Any("error", err), slog.Any("deployRepo", deployRepo), slog.Any("notifyEvt", notifyEvt))
+ }
+ if exist {
+ // fail to delete service
+ slog.Warn("fail to stop deploy instance for insufficient balance", slog.Any("error", err), slog.Any("deployRepo", deployRepo), slog.Any("notifyEvt", notifyEvt))
+ } else {
+ // update database deploy to stopped
+ err = d.store.StopDeploy(ctx, types.ModelRepo, deploy.RepoID, deploy.UserID, deploy.ID)
+ if err != nil {
+ slog.Warn("fail to update deploy status for insufficient balance", slog.Any("error", err), slog.Any("deployRepo", deployRepo), slog.Any("notifyEvt", notifyEvt))
+ }
+ }
+ }
+ return nil
+}
+
+// SubmitEvaluation
+func (d *deployer) SubmitEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) {
+ env := make(map[string]string)
+ env["REVISION"] = "main"
+ env["MODEL_ID"] = req.ModelId
+ env["DATASET_IDS"] = strings.Join(req.Datasets, ",")
+ env["REVISION"] = "main"
+ env["ACCESS_TOKEN"] = req.Token
+ env["HF_ENDPOINT"] = req.DownloadEndpoint
+
+ if req.Hardware.Gpu.Num != "" {
+ env["GPU_NUM"] = req.Hardware.Gpu.Num
+ } else if req.Hardware.Npu.Num != "" {
+ env["NPU_NUM"] = req.Hardware.Npu.Num
+ }
+ templates := []types.ArgoFlowTemplate{}
+ templates = append(templates, types.ArgoFlowTemplate{
+ Name: "evaluation",
+ Env: env,
+ HardWare: req.Hardware,
+ Image: req.Image,
+ },
+ )
+ uniqueFlowName := d.sfNode.Generate().Base36()
+ flowReq := &types.ArgoWorkFlowReq{
+ TaskName: req.TaskName,
+ TaskId: uniqueFlowName,
+ TaskType: req.TaskType,
+ TaskDesc: req.TaskDesc,
+ Image: req.Image,
+ Datasets: req.Datasets,
+ Username: req.Username,
+ UserUUID: req.UserUUID,
+ RepoIds: []string{req.ModelId},
+ Entrypoint: "evaluation",
+ ClusterID: req.ClusterID,
+ Templates: templates,
+ RepoType: req.RepoType,
+ ResourceId: req.ResourceId,
+ ResourceName: req.ResourceName,
+ }
+ if req.ResourceId == 0 {
+ flowReq.ShareMode = true
+ }
+ return d.ir.SubmitWorkFlow(ctx, flowReq)
+}
+func (d *deployer) ListEvaluations(ctx context.Context, username string, per int, page int) (*types.ArgoWorkFlowListRes, error) {
+ return d.ir.ListWorkFlows(ctx, username, per, page)
+}
+
+func (d *deployer) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error {
+ _, err := d.ir.DeleteWorkFlow(ctx, req)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (d *deployer) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) {
+ wf, err := d.ir.GetWorkFlow(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ return wf, err
+}
+
+
+
package imagebuilder
+
+import (
+ "context"
+)
+
+var _ Builder = (*LocalBuilder)(nil)
+
+type LocalBuilder struct{}
+
+func NewLocalBuilder() *LocalBuilder {
+ return &LocalBuilder{}
+}
+
+// Build implements Builder.Build
+func (*LocalBuilder) Build(ctx context.Context, req *BuildRequest) (*BuildResponse, error) {
+ response := &BuildResponse{}
+
+ return response, nil
+}
+
+// Logs implements Builder.Logs
+func (*LocalBuilder) Logs(ctx context.Context, req *LogsRequest) (<-chan string, error) {
+ output := make(chan string, 1)
+ output <- "test build log"
+ return output, nil
+}
+
+// Status implements Builder.Status
+func (*LocalBuilder) Status(ctx context.Context, req *StatusRequest) (*StatusResponse, error) {
+ responses := &StatusResponse{
+ // Code: req.CurrentStatus + 1,
+ Code: 3,
+ Message: "build completed",
+ ImageID: "gradio-test-app:v1.0",
+ }
+ return responses, nil
+}
+
+
+
package imagebuilder
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "time"
+)
+
+var _ Builder = (*RemoteBuilder)(nil)
+
+type RemoteBuilder struct {
+ remote *url.URL
+ client *http.Client
+}
+
+func NewRemoteBuilder(remoteURL string) (*RemoteBuilder, error) {
+ parsedURL, err := url.Parse(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ return &RemoteBuilder{
+ remote: parsedURL,
+ client: http.DefaultClient,
+ }, nil
+}
+
+func (h *RemoteBuilder) Build(ctx context.Context, req *BuildRequest) (*BuildResponse, error) {
+ rel := &url.URL{Path: "/push_data"}
+ u := h.remote.ResolveReference(rel)
+ response, err := h.doRequest(http.MethodPost, u.String(), req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var buildResponse BuildResponse
+ if err := json.NewDecoder(response.Body).Decode(&buildResponse); err != nil {
+ return nil, err
+ }
+
+ return &buildResponse, nil
+}
+
+func (h *RemoteBuilder) Status(ctx context.Context, req *StatusRequest) (*StatusResponse, error) {
+ u := fmt.Sprintf("%s/%s/%s/status?build_id=%s", h.remote, req.OrgName, req.SpaceName, req.BuildID)
+ response, err := h.doRequest(http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var result map[string]int = make(map[string]int)
+ if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
+ return nil, err
+ }
+ var imageID string
+ var code int
+ for k, v := range result {
+ imageID = k
+ code = v
+ break
+ }
+
+ var statusResponse StatusResponse
+ statusResponse.ImageID = imageID
+ statusResponse.Code = code
+ return &statusResponse, nil
+}
+
+func (h *RemoteBuilder) Logs(ctx context.Context, req *LogsRequest) (<-chan string, error) {
+ u := fmt.Sprintf("%s/%s/%s/logs?build_id=%s", h.remote, req.OrgName, req.SpaceName, req.BuildID)
+
+ rc, err := h.doStreamRequest(ctx, http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+
+ return h.readToChannel(rc), nil
+}
+
+func (h *RemoteBuilder) readToChannel(rc io.ReadCloser) <-chan string {
+ output := make(chan string, 2)
+
+ buf := make([]byte, 256)
+ br := bufio.NewReader(rc)
+
+ go func() {
+ for {
+ n, err := br.Read(buf)
+ if err != nil {
+ slog.Info("remote builder log reader aborted", slog.Any("error", err))
+ rc.Close()
+ close(output)
+ break
+ }
+
+ if n > 0 {
+ output <- string(buf[:n])
+ } else {
+ time.Sleep(2 * time.Second)
+ }
+ }
+ }()
+
+ return output
+}
+
+// Helper method to execute the actual HTTP request and read the response.
+func (h *RemoteBuilder) doRequest(method, url string, data interface{}) (*http.Response, error) {
+ var buf io.Reader
+ if data != nil {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ buf = bytes.NewBuffer(jsonData)
+ }
+
+ req, err := http.NewRequest(method, url, buf)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ var errData interface{}
+ err := json.NewDecoder(resp.Body).Decode(&errData)
+ if err != nil {
+ return nil, fmt.Errorf("unexpected http status: %d, error: %w", resp.StatusCode, err)
+ } else {
+ return nil, fmt.Errorf("unexpected http status: %d, error: %v", resp.StatusCode, errData)
+ }
+ }
+
+ return resp, nil
+}
+
+func (h *RemoteBuilder) doStreamRequest(ctx context.Context, method, url string, data interface{}) (io.ReadCloser, error) {
+ var buf io.Reader
+ if data != nil {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ buf = bytes.NewBuffer(jsonData)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, url, buf)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ // req.Header.Set("Accept", "text/event-stream")
+ req.Header.Set("Connection", "keep-alive")
+
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("unexpected http status code:%d", resp.StatusCode)
+ }
+
+ return resp.Body, nil
+}
+
+
+
package imagebuilder
+
+import "io"
+
+type (
+ BuildRequest struct {
+ OrgName string `json:"org_name"`
+ SpaceName string `json:"space_name"`
+
+ Hardware string `json:"hardware"`
+ PythonVersion string `json:"python_version"`
+ SDKType string `json:"sdk"`
+ SDKVersion string `json:"sdk_version"`
+
+ SpaceGitURL string `json:"space_url"`
+ GitRef string `json:"git_ref"`
+ GitUserID string `json:"user_id"`
+ GitAccessToken string `json:"git_access_token"`
+
+ BuildID string `json:"build_id"`
+ FactoryBuild bool `json:"factory_build"`
+ }
+ BuildResponse struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ }
+
+ StatusRequest struct {
+ OrgName string `json:"org_name"`
+ SpaceName string `json:"space_name"`
+ BuildID string `json:"build_id"`
+ // for local builder test only
+ CurrentStatus int
+ }
+
+ StatusResponse struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ ImageID string `json:"image_id"`
+ }
+
+ LogsRequest struct {
+ OrgName string `json:"org_name"`
+ SpaceName string `json:"name"`
+ BuildID string `json:"build_id"`
+ }
+
+ LogsResponse struct {
+ SSEReadCloser io.ReadCloser `json:"sse_read_closer"`
+ }
+)
+
+func (s *StatusResponse) Success() bool {
+ return s.Code == 0
+}
+
+func (s *StatusResponse) Fail() bool {
+ return s.Code == 1
+}
+
+func (s *StatusResponse) Inprogress() bool {
+ return s.Code == 2
+}
+
+
+
package imagerunner
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var _ Runner = (*LocalRunner)(nil)
+
+// Typically this is for local test only
+type LocalRunner struct{}
+
+// InstanceLogs implements Runner.
+func (r *LocalRunner) InstanceLogs(context.Context, *types.InstanceLogsRequest) (<-chan string, error) {
+ output := make(chan string, 1)
+ output <- "test build log"
+ return output, nil
+}
+
+// GetReplica implements Runner.
+func (r *LocalRunner) GetReplica(context.Context, *types.StatusRequest) (*types.ReplicaResponse, error) {
+ return &types.ReplicaResponse{
+ Code: 1,
+ Message: "success",
+ ActualReplica: 0,
+ DesiredReplica: 0,
+ Instances: []types.Instance{},
+ }, nil
+}
+
+// Exist implements Runner.
+func (r *LocalRunner) Exist(context.Context, *types.CheckRequest) (*types.StatusResponse, error) {
+ return &types.StatusResponse{
+ Code: 1,
+ Message: "deploy exist",
+ }, nil
+}
+
+func NewLocalRunner() Runner {
+ return &LocalRunner{}
+}
+
+func (r *LocalRunner) Run(ctx context.Context, req *types.RunRequest) (*types.RunResponse, error) {
+ return &types.RunResponse{
+ Code: 0,
+ Message: "deploy scheduled",
+ }, nil
+}
+
+func (r *LocalRunner) Status(ctx context.Context, req *types.StatusRequest) (*types.StatusResponse, error) {
+ return &types.StatusResponse{
+ Code: common.Running,
+ Message: "deploy success",
+ }, nil
+}
+
+func (r *LocalRunner) StatusAll(ctx context.Context) (map[string]types.StatusResponse, error) {
+ status := make(map[string]types.StatusResponse)
+ status["gradio-test-app"] = types.StatusResponse{Code: 21}
+ status["gradio-test-app-v1-0"] = types.StatusResponse{Code: 20}
+ status["image-123"] = types.StatusResponse{Code: 25}
+ return status, nil
+}
+
+func (r *LocalRunner) Logs(ctx context.Context, req *types.LogsRequest) (<-chan string, error) {
+ output := make(chan string, 1)
+ output <- "test build log"
+ return output, nil
+}
+
+func (r *LocalRunner) Stop(ctx context.Context, req *types.StopRequest) (*types.StopResponse, error) {
+ return &types.StopResponse{}, nil
+}
+
+func (r *LocalRunner) Purge(ctx context.Context, req *types.PurgeRequest) (*types.PurgeResponse, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) ListCluster(ctx context.Context) ([]types.ClusterResponse, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) GetClusterById(ctx context.Context, clusterId string) (*types.ClusterResponse, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) UpdateCluster(ctx context.Context, data *types.ClusterRequest) (*types.UpdateClusterResponse, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) SubmitWorkFlow(ctx context.Context, req *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) ListWorkFlows(ctx context.Context, username string, per, page int) (*types.ArgoWorkFlowListRes, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) {
+ return nil, nil
+}
+
+func (h *LocalRunner) GetWorkFlow(ctx context.Context, req types.EvaluationGetReq) (*types.ArgoWorkFlowRes, error) {
+ return nil, nil
+}
+
+
+
package imagerunner
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "time"
+
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var _ Runner = (*RemoteRunner)(nil)
+
+type RemoteRunner struct {
+ remote *url.URL
+ client *http.Client
+}
+
+func NewRemoteRunner(remoteURL string) (Runner, error) {
+ parsedURL, err := url.Parse(remoteURL)
+ if err != nil {
+ return nil, err
+ }
+ return &RemoteRunner{
+ remote: parsedURL,
+ client: http.DefaultClient,
+ }, nil
+}
+
+func (h *RemoteRunner) Run(ctx context.Context, req *types.RunRequest) (*types.RunResponse, error) {
+ slog.Debug("send request", slog.Any("body", req))
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/run", h.remote, svcName)
+ response, err := h.doRequest(http.MethodPost, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var resp types.RunResponse
+ if err := json.NewDecoder(response.Body).Decode(&resp); err != nil {
+ return nil, err
+ }
+ resp.Message = svcName
+ slog.Debug("call image run", slog.Any("response", resp))
+ return &resp, nil
+}
+
+func (h *RemoteRunner) Stop(ctx context.Context, req *types.StopRequest) (*types.StopResponse, error) {
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/stop", h.remote, svcName)
+ response, err := h.doRequest(http.MethodPost, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var StopResponse types.StopResponse
+ if err := json.NewDecoder(response.Body).Decode(&StopResponse); err != nil {
+ return nil, err
+ }
+
+ return &StopResponse, nil
+}
+
+func (h *RemoteRunner) Purge(ctx context.Context, req *types.PurgeRequest) (*types.PurgeResponse, error) {
+ // u := fmt.Sprintf("%s/%s/stop", h.remote, common.UniqueSpaceAppName(req.OrgName, req.RepoName, req.ID))
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/purge", h.remote, svcName)
+ response, err := h.doRequest(http.MethodDelete, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var purgeResponse types.PurgeResponse
+ if err := json.NewDecoder(response.Body).Decode(&purgeResponse); err != nil {
+ return nil, err
+ }
+
+ return &purgeResponse, nil
+}
+
+func (h *RemoteRunner) Status(ctx context.Context, req *types.StatusRequest) (*types.StatusResponse, error) {
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/status", h.remote, svcName)
+
+ response, err := h.doRequest(http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var statusResponse types.StatusResponse
+ if err := json.NewDecoder(response.Body).Decode(&statusResponse); err != nil {
+ return nil, err
+ }
+
+ return &statusResponse, nil
+}
+
+func (h *RemoteRunner) StatusAll(ctx context.Context) (map[string]types.StatusResponse, error) {
+ u := fmt.Sprintf("%s/api/v1/service/status-all", h.remote)
+ response, err := h.doRequest(http.MethodGet, u, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ statusAll := make(map[string]types.StatusResponse)
+ if err := json.NewDecoder(response.Body).Decode(&statusAll); err != nil {
+ return nil, err
+ }
+
+ return statusAll, nil
+}
+
+func (h *RemoteRunner) Logs(ctx context.Context, req *types.LogsRequest) (<-chan string, error) {
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/logs", h.remote, svcName)
+ slog.Debug("logs request", slog.String("url", u), slog.String("appname", svcName))
+ rc, err := h.doSteamRequest(ctx, http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+
+ return h.readToChannel(rc), nil
+}
+
+func (h *RemoteRunner) Exist(ctx context.Context, req *types.CheckRequest) (*types.StatusResponse, error) {
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/get", h.remote, svcName)
+ response, err := h.doRequest(http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var statusResponse types.StatusResponse
+ if err := json.NewDecoder(response.Body).Decode(&statusResponse); err != nil {
+ return nil, err
+ }
+
+ return &statusResponse, nil
+}
+
+func (h *RemoteRunner) GetReplica(ctx context.Context, req *types.StatusRequest) (*types.ReplicaResponse, error) {
+ svcName := req.SvcName
+ u := fmt.Sprintf("%s/api/v1/service/%s/replica", h.remote, svcName)
+ response, err := h.doRequest(http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var resp types.ReplicaResponse
+ if err := json.NewDecoder(response.Body).Decode(&resp); err != nil {
+ return nil, err
+ }
+
+ return &resp, nil
+}
+
+func (h *RemoteRunner) readToChannel(rc io.ReadCloser) <-chan string {
+ output := make(chan string, 2)
+
+ buf := make([]byte, 256)
+ br := bufio.NewReader(rc)
+
+ go func() {
+ for {
+ n, err := br.Read(buf)
+ if err != nil {
+ slog.Error("remot runner log reader aborted", slog.Any("error", err))
+ rc.Close()
+ close(output)
+ break
+ }
+
+ if n > 0 {
+ output <- string(buf[:n])
+ } else {
+ time.Sleep(2 * time.Second)
+ }
+ }
+ }()
+
+ return output
+}
+
+// Helper method to execute the actual HTTP request and read the response.
+func (h *RemoteRunner) doRequest(method, url string, data interface{}) (*http.Response, error) {
+ var buf io.Reader
+ if data != nil {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ buf = bytes.NewBuffer(jsonData)
+ }
+
+ req, err := http.NewRequest(method, url, buf)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ var errData interface{}
+ err := json.NewDecoder(resp.Body).Decode(&errData)
+ if err != nil {
+ return nil, fmt.Errorf("unexpected http status: %d, error: %w", resp.StatusCode, err)
+ } else {
+ return nil, fmt.Errorf("unexpected http status: %d, error: %v", resp.StatusCode, errData)
+ }
+ }
+
+ return resp, nil
+}
+
+func (h *RemoteRunner) doSteamRequest(ctx context.Context, method, url string, data interface{}) (io.ReadCloser, error) {
+ var buf io.Reader
+ if data != nil {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ buf = bytes.NewBuffer(jsonData)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, url, buf)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Connection", "keep-alive")
+
+ resp, err := h.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("unexpected http status code:%d", resp.StatusCode)
+ }
+
+ return resp.Body, nil
+}
+
+// InstanceLogs implements Runner.
+func (h *RemoteRunner) InstanceLogs(ctx context.Context, req *types.InstanceLogsRequest) (<-chan string, error) {
+ u := fmt.Sprintf("%s/api/v1/service/%s/logs/%s", h.remote, req.SvcName, req.InstanceName)
+ slog.Info("Instance logs request", slog.String("url", u), slog.String("svcname", req.SvcName))
+ rc, err := h.doSteamRequest(ctx, http.MethodGet, u, req)
+ if err != nil {
+ return nil, err
+ }
+
+ return h.readToChannel(rc), nil
+}
+
+func (h *RemoteRunner) ListCluster(ctx context.Context) ([]types.ClusterResponse, error) {
+ url := fmt.Sprintf("%s/api/v1/cluster", h.remote)
+ // Send a GET request to resources runner
+ response, err := h.client.Get(url)
+ if err != nil {
+ fmt.Printf("Error sending request to resoures runner: %s\n", err)
+ return nil, fmt.Errorf("failed to list cluster info, %w", err)
+ }
+ defer response.Body.Close()
+ var resp []types.ClusterResponse
+ if err := json.NewDecoder(response.Body).Decode(&resp); err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
+
+func (h *RemoteRunner) GetClusterById(ctx context.Context, clusterId string) (*types.ClusterResponse, error) {
+ url := fmt.Sprintf("%s/api/v1/cluster/%s", h.remote, clusterId)
+ // Send a GET request to resources runner
+ response, err := h.doRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+ var resp types.ClusterResponse
+ if err := json.NewDecoder(response.Body).Decode(&resp); err != nil {
+ return nil, err
+ }
+ return &resp, nil
+}
+
+func (h *RemoteRunner) UpdateCluster(ctx context.Context, data *types.ClusterRequest) (*types.UpdateClusterResponse, error) {
+ url := fmt.Sprintf("%s/api/v1/cluster/%s", h.remote, data.ClusterID)
+ // Create a new HTTP client with a timeout
+ response, err := h.doRequest(http.MethodPut, url, data)
+ if err != nil {
+ fmt.Printf("Error sending request to k8s cluster: %s\n", err)
+ return nil, fmt.Errorf("failed to update cluster info, %w", err)
+ }
+ defer response.Body.Close()
+ var resp types.UpdateClusterResponse
+ if err := json.NewDecoder(response.Body).Decode(&resp); err != nil {
+ return nil, err
+ }
+
+ return &resp, nil
+}
+
+// submit argo workflow
+func (h *RemoteRunner) SubmitWorkFlow(ctx context.Context, req *types.ArgoWorkFlowReq) (*types.ArgoWorkFlowRes, error) {
+ url := fmt.Sprintf("%s/api/v1/workflows", h.remote)
+ // Create a new HTTP client with a timeout
+ response, err := h.doRequest(http.MethodPost, url, req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to submit evaluation job, %w", err)
+ }
+ defer response.Body.Close()
+
+ var res types.ArgoWorkFlowRes
+ if err := json.NewDecoder(response.Body).Decode(&res); err != nil {
+ return nil, err
+ }
+ return &res, nil
+}
+
+// list workflows
+func (h *RemoteRunner) ListWorkFlows(ctx context.Context, username string, per, page int) (*types.ArgoWorkFlowListRes, error) {
+ url := fmt.Sprintf("%s/api/v1/workflows?username=%s&per=%d&page=%d", h.remote, username, per, page)
+ // Create a new HTTP client with a timeout
+ response, err := h.doRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list evaluation jobs, %w", err)
+ }
+ defer response.Body.Close()
+ var res types.ArgoWorkFlowListRes
+ if err := json.NewDecoder(response.Body).Decode(&res); err != nil {
+ return nil, err
+ }
+ return &res, nil
+}
+
+// delete workflow
+func (h *RemoteRunner) DeleteWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*httpbase.R, error) {
+ url := fmt.Sprintf("%s/api/v1/workflows/%d", h.remote, req.ID)
+ // Create a new HTTP client with a timeout
+ response, err := h.doRequest(http.MethodDelete, url, req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to delete evaluation job, %w", err)
+ }
+ defer response.Body.Close()
+ var res httpbase.R
+ err = json.NewDecoder(response.Body).Decode(&res)
+ if err != nil {
+ return nil, err
+ }
+ return &res, nil
+}
+
+func (h *RemoteRunner) GetWorkFlow(ctx context.Context, req types.ArgoWorkFlowDeleteReq) (*types.ArgoWorkFlowRes, error) {
+ url := fmt.Sprintf("%s/api/v1/workflows/%d", h.remote, req.ID)
+ response, err := h.doRequest(http.MethodGet, url, req)
+ if err != nil {
+ return nil, err
+ }
+ defer response.Body.Close()
+
+ var res types.ArgoWorkFlowRes
+ if err := json.NewDecoder(response.Body).Decode(&res); err != nil {
+ return nil, err
+ }
+ return &res, nil
+}
+
+
+
package deploy
+
+import (
+ "fmt"
+ "time"
+
+ "opencsg.com/csghub-server/builder/deploy/imagebuilder"
+ "opencsg.com/csghub-server/builder/deploy/imagerunner"
+ "opencsg.com/csghub-server/builder/deploy/scheduler"
+)
+
+var (
+ fifoScheduler scheduler.Scheduler
+ defaultDeployer Deployer
+)
+
+func Init(c DeployConfig) error {
+ // ib := imagebuilder.NewLocalBuilder()
+ ib, err := imagebuilder.NewRemoteBuilder(c.ImageBuilderURL)
+ if err != nil {
+ panic(fmt.Errorf("failed to create image builder:%w", err))
+ }
+ ir, err := imagerunner.NewRemoteRunner(c.ImageRunnerURL)
+ if err != nil {
+ panic(fmt.Errorf("failed to create image runner:%w", err))
+ }
+
+ fifoScheduler = scheduler.NewFIFOScheduler(ib, ir, c.SpaceDeployTimeoutInMin, c.ModelDeployTimeoutInMin, c.ModelDownloadEndpoint, c.PublicRootDomain)
+ deployer, err := newDeployer(fifoScheduler, ib, ir, c)
+ if err != nil {
+ return fmt.Errorf("failed to create deployer:%w", err)
+ }
+
+ deployer.internalRootDomain = c.InternalRootDomain
+ defaultDeployer = deployer
+ return nil
+}
+
+func NewDeployer() Deployer {
+ return defaultDeployer
+}
+
+type DeployConfig struct {
+ ImageBuilderURL string
+ ImageRunnerURL string
+ MonitorInterval time.Duration
+ InternalRootDomain string
+ SpaceDeployTimeoutInMin int
+ ModelDeployTimeoutInMin int
+ ModelDownloadEndpoint string
+ ChargingEnable bool
+ AutoCleanInstance bool
+ PublicRootDomain string
+ IsMasterHost bool
+}
+
+
+
package deploy
+
+type MultiLogReader struct {
+ buildLogs <-chan string
+ runLogs <-chan string
+}
+
+func NewMultiLogReader(logs ...<-chan string) *MultiLogReader {
+ r := new(MultiLogReader)
+ r.buildLogs = logs[0]
+ r.runLogs = logs[1]
+ return r
+}
+
+func (r *MultiLogReader) BuildLog() <-chan string {
+ return r.buildLogs
+}
+
+func (r *MultiLogReader) RunLog() <-chan string {
+ return r.runLogs
+}
+
+
+
package scheduler
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/builder/deploy/imagebuilder"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+// BuilderRunner defines a docker image building task
+type BuilderRunner struct {
+ repo *RepoInfo
+ task *database.DeployTask
+ ib imagebuilder.Builder
+ deployStore database.DeployTaskStore
+ tokenStore database.AccessTokenStore
+}
+
+func NewBuidRunner(b imagebuilder.Builder, r *RepoInfo, t *database.DeployTask) Runner {
+ return &BuilderRunner{
+ repo: r,
+ task: t,
+ ib: b,
+ deployStore: database.NewDeployTaskStore(),
+ tokenStore: database.NewAccessTokenStore(),
+ }
+}
+
+func (t *BuilderRunner) makeBuildRequest() (*imagebuilder.BuildRequest, error) {
+ token, err := t.tokenStore.FindByUID(context.Background(), t.task.Deploy.UserID)
+ if err != nil {
+ return nil, fmt.Errorf("cant get git access token:%w", err)
+ }
+ fields := strings.Split(t.repo.Path, "/")
+ sdkVer := ""
+ if t.repo.SdkVersion == "" {
+ slog.Warn("Use SDK default version", slog.Any("repository path", t.repo.Path))
+ if t.repo.Sdk == GRADIO.Name {
+ sdkVer = GRADIO.Version
+ } else if t.repo.Sdk == STREAMLIT.Name {
+ sdkVer = STREAMLIT.Version
+ }
+ } else {
+ sdkVer = t.repo.SdkVersion
+ }
+ return &imagebuilder.BuildRequest{
+ OrgName: fields[0],
+ SpaceName: fields[1],
+ Hardware: t.parseHardware(t.task.Deploy.Hardware),
+ // PythonVersion: t.space.PythonVersion,
+ PythonVersion: "3.10",
+ // SDKType: "gradio",
+ // SDKVersion: "3.37.0",
+ SDKType: t.repo.Sdk,
+ SDKVersion: sdkVer,
+ SpaceGitURL: t.repo.HTTPCloneURL,
+ GitRef: t.task.Deploy.GitBranch,
+ GitUserID: token.User.Username,
+ GitAccessToken: token.Token,
+ BuildID: strconv.FormatInt(t.task.DeployID, 10),
+ FactoryBuild: false,
+ }, nil
+}
+
+func (t *BuilderRunner) parseHardware(intput string) string {
+ if strings.Contains(intput, "GPU") || strings.Contains(intput, "NVIDIA") {
+ return "gpu"
+ }
+
+ return "cpu"
+}
+
+// Run call image builder service to build a docker image
+func (t *BuilderRunner) Run(ctx context.Context) error {
+ slog.Info("run image build task", slog.Int64("deplopy_task_id", t.task.ID))
+
+ if t.task.Status == buildPending {
+ req, err := t.makeBuildRequest()
+ if err != nil {
+ return fmt.Errorf("make build request failed: %w", err)
+ }
+ slog.Debug("make build request", slog.Any("req", req))
+ resp, err := t.ib.Build(context.Background(), req)
+ if err != nil {
+ // TODO:return retryable error
+ return fmt.Errorf("call image builder failed: %w", err)
+ }
+ if resp.Code != 0 {
+ // job failed
+ return fmt.Errorf("image builder reported error,code:%d,msg:%s", resp.Code, resp.Message)
+ }
+
+ t.buildInProgress()
+ }
+
+ // keep checking build status
+ for {
+ fields := strings.Split(t.repo.Path, "/")
+ req := &imagebuilder.StatusRequest{
+ OrgName: fields[0],
+ SpaceName: fields[1],
+ BuildID: strconv.FormatInt(t.task.DeployID, 10),
+ }
+ resp, err := t.ib.Status(context.Background(), req)
+ slog.Debug("image builder called", slog.Any("resp", resp), slog.Any("error", err))
+ if err != nil {
+ // return -1, fmt.Errorf("failed to call builder status api,%w", err)
+ slog.Error("failed to call builder status api", slog.Any("error", err), slog.Any("task", t))
+ // wait before next check
+ time.Sleep(10 * time.Second)
+ continue
+ }
+ switch {
+ case resp.Inprogress():
+ // wait before next check
+ time.Sleep(10 * time.Second)
+ continue
+ case resp.Success():
+ slog.Info("image build succeeded", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID))
+ t.buildSuccess(*resp)
+
+ return nil
+ case resp.Fail():
+ slog.Info("image build failed", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID))
+ t.buildFailed()
+
+ return nil
+ }
+ }
+}
+
+func (t *BuilderRunner) buildInProgress() {
+ t.task.Status = buildInProgress
+ t.task.Message = "build in progress"
+ // change to building status
+ t.task.Deploy.Status = common.Building
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.deployStore.UpdateInTx(ctx, []string{"status"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `Building`", "error", err)
+ }
+}
+
+func (t *BuilderRunner) buildSuccess(resp imagebuilder.StatusResponse) {
+ t.task.Status = buildSucceed
+ t.task.Message = "build succeeded"
+ // change to building status
+ t.task.Deploy.Status = common.BuildSuccess
+ t.task.Deploy.ImageID = resp.ImageID
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.deployStore.UpdateInTx(ctx, []string{"status", "image_id"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `BuildSuccess`", "error", err)
+ }
+}
+
+func (t *BuilderRunner) buildFailed() {
+ t.task.Status = buildFailed
+ t.task.Message = "build failed"
+ // change to building status
+ t.task.Deploy.Status = common.BuildFailed
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.deployStore.UpdateInTx(ctx, []string{"status"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `BuildFailed`", "error", err)
+ }
+}
+
+func (t *BuilderRunner) WatchID() int64 { return t.task.ID }
+
+
+
package scheduler
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/builder/deploy/imagerunner"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type DeployTimeout struct {
+ deploySpaceTimeoutInMin int
+ deployModelTimeoutInMin int
+}
+
+// DeployRunner defines a k8s image running task
+type DeployRunner struct {
+ repo *RepoInfo
+ task *database.DeployTask
+ ir imagerunner.Runner
+ store database.DeployTaskStore
+ tokenStore database.AccessTokenStore
+ deployStartTime time.Time
+ deployTimeout *DeployTimeout
+ modelDownloadEndpoint string
+ urs database.UserResourcesStore
+ publicDomain string
+}
+
+func NewDeployRunner(ir imagerunner.Runner, r *RepoInfo, t *database.DeployTask, dto *DeployTimeout, mdep, publicDomain string) Runner {
+ return &DeployRunner{
+ repo: r,
+ task: t,
+ ir: ir,
+ store: database.NewDeployTaskStore(),
+ deployStartTime: time.Now(),
+ deployTimeout: dto,
+ tokenStore: database.NewAccessTokenStore(),
+ modelDownloadEndpoint: mdep,
+ publicDomain: publicDomain,
+ urs: database.NewUserResourcesStore(),
+ }
+}
+
+// Run call k8s image runner service to run a docker image
+func (t *DeployRunner) Run(ctx context.Context) error {
+ slog.Info("run image deploy task", slog.Int64("deplopy_task_id", t.task.ID))
+
+ // keep checking deploy status
+ for {
+ if t.task.Status == deployPending {
+ req, err := t.makeDeployRequest()
+ if err != nil {
+ return fmt.Errorf("fail to make deploy request: %w", err)
+ }
+ if req.ImageID == "" {
+ time.Sleep(5 * time.Second)
+ continue
+ }
+ slog.Debug("After build deploy request", slog.Any("req", req))
+ resp, err := t.ir.Run(ctx, req)
+ if err != nil {
+ // TODO:return retryable error
+ return fmt.Errorf("call image runner failed: %w", err)
+ }
+ // record deploy id to user resource if deploy has order id
+ if req.OrderDetailID != 0 {
+ ur, err := t.urs.FindUserResourcesByOrderDetailId(ctx, req.UserID, req.OrderDetailID)
+ if err != nil {
+ return fmt.Errorf("fail to find user resource by order detail id: %w", err)
+ }
+ ur.DeployId = req.DeployID
+ err = t.urs.UpdateDeployId(ctx, ur)
+ if err != nil {
+ return fmt.Errorf("fail to update deploy id for user resource: %w", err)
+ }
+ }
+
+ t.deployInProgress(resp.Message)
+ // record time of create knative service
+ t.deployStartTime = time.Now()
+ }
+
+ fields := strings.Split(t.repo.Path, "/")
+
+ targetID := t.task.Deploy.SpaceID
+ if t.task.Deploy.SpaceID == 0 {
+ targetID = t.task.Deploy.ID // support model deploy with multi-instance
+ }
+ req := &types.StatusRequest{
+ ID: targetID,
+ OrgName: fields[0],
+ RepoName: fields[1],
+ SvcName: t.task.Deploy.SvcName,
+ ClusterID: t.task.Deploy.ClusterID,
+ NeedDetails: true, // check status of both knative and its pods
+ }
+ timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ resp, err := t.ir.Status(timeoutCtx, req)
+ cancel()
+ if err != nil {
+ // return -1, fmt.Errorf("failed to call builder status api,%w", err)
+ slog.Error("failed to call runner status api", slog.Any("error", err), slog.Any("task", t.task))
+ // wait before next check
+ time.Sleep(10 * time.Second)
+ continue
+ }
+
+ if resp.DeployID > t.task.DeployID {
+ t.deployFailed(fmt.Sprintf("cancel by new deploy:%d", resp.DeployID))
+ return nil
+ }
+ switch resp.Code {
+ case common.Deploying:
+ duration := time.Since(t.deployStartTime).Minutes()
+ limitTime := t.deployTimeout.deploySpaceTimeoutInMin
+ if t.task.Deploy.SpaceID == 0 && t.task.Deploy.ModelID > 0 {
+ limitTime = t.deployTimeout.deployModelTimeoutInMin
+ }
+ if duration >= float64(limitTime) {
+ // space or model deploy duration is greater than timeout defined in env (default is 30 mins)
+ slog.Warn("Space or Model is going to be undeploy due to timeout of deploying", slog.Any("duration", duration), slog.Any("timeout", limitTime), slog.Any("namespace", fields[0]), slog.Any("repoName", fields[1]))
+ return t.cancelDeploy(ctx, fields[0], fields[1])
+ }
+ t.deployInProgress("")
+ // wait before next check
+ time.Sleep(10 * time.Second)
+ case common.DeployFailed:
+ slog.Error("image deploy failed", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID), slog.Any("resp", resp))
+ t.deployFailed(resp.Message)
+
+ return fmt.Errorf("deploy failed, resp msg:%s", resp.Message)
+ case common.Startup:
+ slog.Info("image deploy success", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID))
+ t.deploySuccess()
+ // wait before next check
+ time.Sleep(10 * time.Second)
+
+ case common.Running:
+ slog.Info("image running", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID))
+ t.running(resp.Endpoint)
+
+ return nil
+ case common.RunTimeError:
+ slog.Error("image runtime erro", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID))
+ t.runtimeError(resp.Message)
+
+ return fmt.Errorf("runtime error, resp msg:%s", resp.Message)
+ default:
+ slog.Error("unknown image status", slog.String("repo_name", t.repo.Name), slog.Any("deplopy_task_id", t.task.ID),
+ slog.Int("status", resp.Code))
+ return fmt.Errorf("unknown image status, resp msg:%s", resp.Message)
+ }
+ }
+}
+
+func (t *DeployRunner) WatchID() int64 { return t.task.ID }
+
+func (t *DeployRunner) deployInProgress(svcName string) {
+ t.task.Status = deploying
+ t.task.Message = "deploy in progress"
+ // change to building status
+ t.task.Deploy.Status = common.Deploying
+ if len(svcName) > 0 {
+ t.task.Deploy.SvcName = svcName
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.store.UpdateInTx(ctx, []string{"status", "svc_name"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `Deploying`", "error", err)
+ }
+}
+
+func (t *DeployRunner) deploySuccess() {
+ t.task.Status = deployStartUp
+ t.task.Message = "deploy succeeded, wati for startup"
+ // change to building status
+ t.task.Deploy.Status = common.Startup
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.store.UpdateInTx(ctx, []string{"status"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `Startup`", "error", err)
+ }
+}
+
+func (t *DeployRunner) deployFailed(msg string) {
+ t.task.Status = deployFailed
+ t.task.Message = msg
+ // change to building status
+ t.task.Deploy.Status = common.DeployFailed
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.store.UpdateInTx(ctx, []string{"status"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `DeployFailed`", "error", err)
+ }
+}
+
+func (t *DeployRunner) running(endpoint string) {
+ t.task.Status = deployRunning
+ t.task.Message = "running"
+ // change to building status
+ t.task.Deploy.Status = common.Running
+ t.task.Deploy.Endpoint = endpoint
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.store.UpdateInTx(ctx, []string{"status", "endpoint"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `Running`", "error", err)
+ }
+}
+
+func (t *DeployRunner) runtimeError(msg string) {
+ t.task.Status = deployRunTimeError
+ t.task.Message = msg
+ // change to building status
+ t.task.Deploy.Status = common.RunTimeError
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if err := t.store.UpdateInTx(ctx, []string{"status"}, []string{"status", "message"}, t.task.Deploy, t.task); err != nil {
+ slog.Error("failed to change deploy status to `RunTimeError`", "error", err)
+ }
+}
+
+func (t *DeployRunner) makeDeployRequest() (*types.RunRequest, error) {
+ token, err := t.tokenStore.FindByUID(context.Background(), t.task.Deploy.UserID)
+ if err != nil {
+ return nil, fmt.Errorf("cant get git access token:%w", err)
+ }
+ fields := strings.Split(t.repo.Path, "/")
+ deploy, err := t.store.GetDeployByID(context.Background(), t.task.DeployID)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get deploy with error :%w", err)
+ }
+
+ annoMap, err := common.JsonStrToMap(deploy.Annotation)
+ if err != nil {
+ slog.Error("deploy annotation is invalid json data", slog.Any("Annotation", deploy.Annotation))
+ return nil, err
+ }
+ annoMap[types.ResDeployID] = fmt.Sprintf("%v", deploy.ID)
+
+ envMap, err := common.JsonStrToMap(deploy.Env)
+ if err != nil {
+ slog.Error("deploy env is invalid json data", slog.Any("env", deploy.Env))
+ return nil, err
+ }
+
+ var hardware = types.HardWare{}
+ err = json.Unmarshal([]byte(deploy.Hardware), &hardware)
+ if err != nil {
+ slog.Error("deploy hardware is invalid format", slog.Any("hardware", deploy.Hardware))
+ return nil, err
+ }
+
+ // for space and models
+ envMap["HTTPCloneURL"] = t.getHttpCloneURLWithToken(t.repo.HTTPCloneURL, token.Token)
+ envMap["ACCESS_TOKEN"] = token.Token
+ envMap["REPO_ID"] = t.repo.Path // "namespace/name"
+ envMap["REVISION"] = deploy.GitBranch // branch
+ if hardware.Gpu.Num != "" {
+ envMap["GPU_NUM"] = hardware.Gpu.Num
+ } else if hardware.Npu.Num != "" {
+ envMap["NPU_NUM"] = hardware.Npu.Num
+ }
+
+ if deploy.SpaceID > 0 {
+ // sdk port for space
+ if t.repo.Sdk == GRADIO.Name {
+ envMap["port"] = GRADIO.Port
+ } else if t.repo.Sdk == STREAMLIT.Name {
+ envMap["port"] = STREAMLIT.Port
+ } else if t.repo.Sdk == NGINX.Name {
+ envMap["port"] = NGINX.Port
+ } else {
+ envMap["port"] = "8080"
+ }
+ }
+
+ if deploy.Type == types.InferenceType || deploy.Type == types.ServerlessType {
+ // runtime framework port for model
+ envMap["port"] = strconv.Itoa(deploy.ContainerPort)
+ envMap["HF_ENDPOINT"] = t.modelDownloadEndpoint // "https://hub-stg.opencsg.com/"
+ envMap["HF_HUB_OFFLINE"] = "1"
+ }
+
+ if deploy.Type == types.FinetuneType {
+ envMap["port"] = strconv.Itoa(deploy.ContainerPort)
+ envMap["HF_ENDPOINT"], _ = url.JoinPath(t.modelDownloadEndpoint, "hf")
+ envMap["HF_TOKEN"] = token.Token
+ envMap["USE_CSGHUB_MODEL"] = "1"
+ envMap["USE_CSGHUB_DATASET"] = "1"
+ }
+
+ if t.publicDomain == "" {
+ if deploy.Type == types.FinetuneType {
+ envMap["CONTEXT_PATH"] = "/endpoint/" + deploy.SvcName
+ }
+ if deploy.Type == types.SpaceType {
+ envMap["GRADIO_ROOT_PATH"] = "/endpoint/" + deploy.SvcName
+ envMap["STREAMLIT_SERVER_BASE_URL_PATH"] = "/endpoint/" + deploy.SvcName
+ }
+
+ }
+
+ targetID := deploy.SpaceID
+ // deployID is unique for space and model
+ if deploy.SpaceID == 0 && deploy.ModelID > 0 {
+ targetID = deploy.ID // support model deploy with multi-instance
+ }
+
+ return &types.RunRequest{
+ ID: targetID,
+ OrgName: fields[0],
+ RepoName: fields[1],
+ RepoType: t.repo.RepoType,
+ UserName: t.repo.UserName,
+ Annotation: annoMap,
+ Hardware: hardware,
+ Env: envMap,
+ GitPath: deploy.GitPath,
+ GitRef: deploy.GitBranch,
+ ImageID: deploy.ImageID,
+ DeployID: deploy.ID,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ Accesstoken: token.Token,
+ ClusterID: deploy.ClusterID,
+ SvcName: deploy.SvcName,
+ DeployType: deploy.Type,
+ UserID: deploy.UserUUID,
+ Sku: deploy.SKU,
+ OrderDetailID: deploy.OrderDetailID,
+ }, nil
+}
+
+func (t *DeployRunner) cancelDeploy(ctx context.Context, orgName, repoName string) error {
+ targetID := t.task.Deploy.SpaceID
+ if t.task.Deploy.SpaceID == 0 {
+ // support model deploy with multi-instance
+ targetID = t.task.Deploy.ID
+ }
+ stopReq := &types.StopRequest{
+ ID: targetID,
+ OrgName: orgName,
+ RepoName: repoName,
+ SvcName: t.task.Deploy.SvcName,
+ }
+ _, err := t.ir.Stop(ctx, stopReq)
+ if err != nil {
+ return fmt.Errorf("fail to undeploy space/model with err: %v", err)
+ }
+ t.deployFailed("space/model deploy timeout")
+ return nil
+}
+
+func (t *DeployRunner) getHttpCloneURLWithToken(httpCloneUrl, token string) string {
+ num := strings.Index(httpCloneUrl, "://")
+ if num > -1 {
+ return fmt.Sprintf("%s%s@%s", httpCloneUrl[0:num+3], token, httpCloneUrl[num+3:])
+ }
+ return httpCloneUrl
+}
+
+
+
package scheduler
+
+import (
+ "context"
+ "log/slog"
+ "time"
+)
+
+type Runner interface {
+ Run(context.Context) error
+ // WatchID is the unique ID for monitor service to watch the running progress
+ WatchID() int64
+}
+
+var (
+ _ Runner = (*BuilderRunner)(nil)
+ _ Runner = (*DeployRunner)(nil)
+)
+
+type sleepTask struct {
+ du time.Duration
+}
+
+func (t *sleepTask) Run(ctx context.Context) error {
+ slog.Debug("sleeping task running", slog.Duration("time", t.du))
+ time.Sleep(t.du)
+ return nil
+}
+func (t *sleepTask) WatchID() int64 { return 0 }
+
+const cancelled = -1
+
+const (
+ buildPending = 0
+ buildInProgress = 1
+ buildFailed = 2
+ buildSucceed = 3
+ BuildSkip = 4 // export for other package
+)
+
+// sub deploy task status
+const (
+ deployPending = 0
+ deploying = 1
+ deployFailed = 2
+ deployStartUp = 3
+ deployRunning = 4
+ deployRunTimeError = 5
+)
+
+
+
package scheduler
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "log/slog"
+ "sync"
+ "time"
+
+ "opencsg.com/csghub-server/builder/deploy/imagebuilder"
+ "opencsg.com/csghub-server/builder/deploy/imagerunner"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+type Scheduler interface {
+ Run() error
+ Queue(deployTaskID int64) error
+}
+
+// a Scheduler will run tasks in their arrival order
+type FIFOScheduler struct {
+ timeout time.Duration
+ // parallel running tasks
+ tasks chan Runner
+ last *database.DeployTask
+
+ store database.DeployTaskStore
+ spaceStore database.SpaceStore
+ modelStore database.ModelStore
+ spaceResourcesStore database.SpaceResourceStore
+ ib imagebuilder.Builder
+ ir imagerunner.Runner
+
+ nextLock *sync.Mutex
+ spaceDeployTimeoutInMin int
+ modelDeployTimeoutInMin int
+ modelDownloadEndpoint string
+ PublicRootDomain string
+ config *config.Config
+}
+
+func NewFIFOScheduler(ib imagebuilder.Builder, ir imagerunner.Runner, sdt, mdt int, mdep, prd string) Scheduler {
+ s := &FIFOScheduler{}
+ // TODO:allow config
+ s.timeout = 30 * time.Minute
+ s.store = database.NewDeployTaskStore()
+ s.spaceStore = database.NewSpaceStore()
+ s.modelStore = database.NewModelStore()
+ s.spaceResourcesStore = database.NewSpaceResourceStore()
+ // allow concurrent deployment tasks
+ s.tasks = make(chan Runner, 100)
+ // s.ib = imagebuilder.NewLocalBuilder()
+ // s.ir = imagerunner.NewLocalRunner()
+ s.ib = ib
+ s.ir = ir
+ s.nextLock = &sync.Mutex{}
+ s.spaceDeployTimeoutInMin = sdt
+ s.modelDeployTimeoutInMin = mdt
+ s.modelDownloadEndpoint = mdep
+ s.PublicRootDomain = prd
+ s.config, _ = config.LoadConfig()
+ return s
+}
+
+// Run will load tasks and run them currently
+func (rs *FIFOScheduler) Run() error {
+ slog.Info("FIFOScheduler run started")
+
+ go func() {
+ for count := 0; count <= cap(rs.tasks); count++ {
+ _, err := rs.next()
+ if err != nil {
+ slog.Error("failed to get next task", "error", err)
+ continue
+ }
+ }
+ }()
+
+ slog.Debug("scheduler try to loop through tasks channel")
+ for t := range rs.tasks {
+ go func(t Runner) {
+ slog.Debug("dequeue a task to run", slog.Any("task", t.WatchID()))
+ ctx, cancel := context.WithTimeout(context.Background(), rs.timeout)
+ defer cancel()
+
+ if err := t.Run(ctx); err != nil {
+ slog.Error("failed to run task", slog.Any("error", err), slog.Any("task", t.WatchID()))
+ rs.failDeployFollowingTasks(t.WatchID(), err.Error())
+ }
+
+ _, _ = rs.next()
+ }(t)
+ }
+
+ return nil
+}
+
+func (rs *FIFOScheduler) Queue(deployTaskID int64) error {
+ // simply trigger next task
+ _, err := rs.next()
+
+ return err
+}
+
+// run next task
+func (rs *FIFOScheduler) next() (Runner, error) {
+ rs.nextLock.Lock()
+ slog.Debug("FIFOScheduler try to get next task", slog.Any("last", rs.last))
+ defer rs.nextLock.Unlock()
+
+ var (
+ deployTask *database.DeployTask
+ t Runner
+ err error
+ )
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if rs.last == nil {
+ deployTask, err = rs.store.GetNewTaskFirst(ctx)
+ slog.Debug("GetNewTaskFirst", slog.Any("deploy_task", deployTask), slog.Any("error", err))
+ } else {
+ deployTask, err = rs.store.GetNewTaskAfter(ctx, rs.last.ID)
+ slog.Debug("GetNewTaskAfter", slog.Any("deploy_task", deployTask), slog.Any("last", rs.last.ID), slog.Any("error", err))
+ }
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ slog.Debug("no more tasks to run, schedule a sleeping task")
+ // using a sleep task to pause the scheduler
+ } else {
+ slog.Error("FIFOScheduler cannot get next task by db error", slog.Any("error", err))
+ }
+
+ t = &sleepTask{
+ du: 5 * time.Second,
+ }
+ rs.tasks <- t
+ return t, nil
+ }
+
+ var repo RepoInfo
+
+ if deployTask.Deploy.SpaceID > 0 {
+ // handle space
+ var s *database.Space
+ s, err = rs.spaceStore.ByID(ctx, deployTask.Deploy.SpaceID)
+ if err == nil {
+ repoCloneInfo := common.BuildCloneInfo(rs.config, s.Repository)
+ repo.Path = s.Repository.Path
+ repo.Name = s.Repository.Name
+ repo.Sdk = s.Sdk
+ repo.SdkVersion = s.SdkVersion
+ repo.HTTPCloneURL = repoCloneInfo.HTTPCloneURL
+ repo.SpaceID = s.ID
+ repo.RepoID = s.Repository.ID
+ repo.UserName = s.Repository.User.Username
+ repo.DeployID = deployTask.Deploy.ID
+ repo.ModelID = 0
+ repo.RepoType = string(types.SpaceRepo)
+ }
+ } else if deployTask.Deploy.ModelID > 0 {
+ // handle model
+ var m *database.Model
+ m, err = rs.modelStore.ByID(ctx, deployTask.Deploy.ModelID)
+ if err == nil {
+ repo.Path = m.Repository.Path
+ repo.Name = m.Repository.Name
+ repo.ModelID = m.ID
+ repo.RepoID = m.Repository.ID
+ repo.UserName = m.Repository.User.Username
+ repo.DeployID = deployTask.Deploy.ID
+ repo.SpaceID = 0
+ repo.RepoType = string(types.ModelRepo)
+ }
+ }
+
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ slog.Warn("cancel deploy task as repo not found", slog.Any("deploy_task", deployTask))
+ // mark task as cancelled
+ deployTask.Status = cancelled
+ deployTask.Message = "repo not found"
+ err = rs.store.UpdateDeployTask(ctx, deployTask)
+ if err != nil {
+ slog.Error("update deploy task failed", "error", err)
+ }
+ }
+ t = &sleepTask{
+ du: 5 * time.Second,
+ }
+ rs.last = deployTask
+ rs.tasks <- t
+ return t, nil
+ }
+ // for build task
+ if deployTask.TaskType == 0 {
+ t = NewBuidRunner(rs.ib, &repo, deployTask)
+ } else {
+ t = NewDeployRunner(rs.ir, &repo, deployTask,
+ &DeployTimeout{
+ deploySpaceTimeoutInMin: rs.spaceDeployTimeoutInMin,
+ deployModelTimeoutInMin: rs.modelDeployTimeoutInMin,
+ },
+ rs.modelDownloadEndpoint,
+ rs.PublicRootDomain,
+ )
+ }
+
+ rs.last = deployTask
+ rs.tasks <- t
+ slog.Info("enqueue next task", slog.Any("task", t.WatchID()))
+ return t, err
+}
+
+func (rs *FIFOScheduler) failDeployFollowingTasks(deploytaskID int64, reason string) {
+ slog.Info("scheduler fail following tasks", slog.Any("deploy_task_id", deploytaskID))
+ t, _ := rs.store.GetDeployTask(context.Background(), deploytaskID)
+
+ dps, err := rs.store.GetDeployTasksOfDeploy(context.Background(), t.DeployID)
+ if err != nil {
+ slog.Error("failed to get tasks of deploy when check build status", slog.Any("error", err),
+ slog.Int64("deploy_id", t.DeployID))
+ return
+ }
+
+ // update following tasks to be failed to stop scheduler to run it
+ for _, dp := range dps {
+ // fail current task
+ if dp.ID == t.ID {
+ dp.Status = buildFailed
+ dp.Message = reason
+ continue
+ }
+ // tasks after current task
+ if dp.ID > t.ID {
+ dp.Status = cancelled
+ dp.Message = "cancel as previous task failed"
+ }
+ }
+ if err := rs.store.UpdateInTx(context.Background(), nil, []string{"status", "message"}, nil, dps...); err != nil {
+ slog.Error("failed update deploy status to `BuildFailed`", slog.Int64("deploy_task_id", t.ID), "error", err)
+ return
+ }
+}
+
+
+
package event
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/nats-io/nats.go/jetstream"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/mq"
+)
+
+var (
+ CSGHubServerDurableConsumerName string = "NoBalanceConsumerForCSGHubServer"
+ CSGHubOrderExpriedConsumerName string = "OrderExpiredConsumerForCSGHubServer"
+ DefaultEventPublisher EventPublisher
+)
+
+type EventPublisher struct {
+ Connector mq.MessageQueue
+ SyncInterval int //in minutes
+}
+
+// NewNatsConnector initializes a new connection to the NATS server
+func InitEventPublisher(cfg *config.Config, natsHandler mq.MessageQueue) error {
+ var handler mq.MessageQueue
+ var err error
+ if natsHandler == nil {
+ handler, err = mq.Init(cfg)
+ if err != nil {
+ return err
+ }
+ } else {
+ handler = natsHandler
+ }
+ DefaultEventPublisher = EventPublisher{
+ Connector: handler,
+ SyncInterval: cfg.Event.SyncInterval,
+ }
+ return nil
+}
+
+func (ec *EventPublisher) CreateNoBalanceConsumer() (jetstream.Consumer, error) {
+ return ec.Connector.BuildNotifyConsumerWithName(CSGHubServerDurableConsumerName)
+}
+
+func (ec *EventPublisher) CreateOrderExporedConsumer() (jetstream.Consumer, error) {
+ return ec.Connector.BuildOrderConsumerWithName(CSGHubOrderExpriedConsumerName)
+}
+
+// Publish a message to the specified subject
+func (ec *EventPublisher) PublishMeteringEvent(message []byte) error {
+ var err error
+ for i := 0; i < 3; i++ {
+ err = ec.Connector.VerifyMeteringStream()
+ if err != nil {
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ err = ec.Connector.PublishMeterDurationData(message)
+ if err == nil {
+ break
+ }
+ time.Sleep(1 * time.Second)
+ }
+
+ if err != nil {
+ return fmt.Errorf("failed to publish metering event for 3 retries, %w", err)
+ }
+
+ return nil
+}
+
+func (ec *EventPublisher) PublishRechargeEvent(message []byte) error {
+ var err error
+ for i := 0; i < 3; i++ {
+ err = ec.Connector.VerifyRechargeStream()
+ if err != nil {
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ err = ec.Connector.PublishRechargeDurationData(message)
+ if err == nil {
+ break
+ }
+ time.Sleep(1 * time.Second)
+ }
+
+ if err != nil {
+ return fmt.Errorf("failed to publish recharge event for 3 retries, %w", err)
+ }
+
+ return nil
+}
+
+
+
package geo
+
+import (
+ "fmt"
+ "net/netip"
+ "net/url"
+)
+
+// CDNUrl replace the host of originalUrl to the cdn by user client ip geo location
+//
+// if no matched cdn domain, return the original url
+func CDNUrlString(clientIP string, originalUrl string) (string, error) {
+ parsedUrl, err := url.Parse(originalUrl)
+ if err != nil {
+ return originalUrl, fmt.Errorf("failed to parse original url, %w", err)
+ }
+ cdnUrl, err := CDNUrl(clientIP, parsedUrl)
+ if err != nil {
+ return originalUrl, fmt.Errorf("failed to get cdn url, %w", err)
+ }
+ return cdnUrl.String(), nil
+}
+
+func CDNUrl(clientIP string, originalUrl *url.URL) (*url.URL, error) {
+ if len(cityToCdnDomain) == 0 {
+ return originalUrl, nil
+ }
+
+ cdnDomain, err := getCdnDomainByIp(clientIP)
+ if err != nil {
+ return originalUrl, fmt.Errorf("failed to get cdn domain by ip, %w", err)
+ }
+ if cdnDomain != "" {
+ originalUrl.Host = cdnDomain
+ }
+ return originalUrl, nil
+}
+
+func getCdnDomainByIp(clientIP string) (string, error) {
+ ip, err := netip.ParseAddr(clientIP)
+ if err != nil {
+ return "", fmt.Errorf("failed to parse client ip, %w", err)
+ }
+ if ip.IsPrivate() || ip.IsLoopback() || ip.IsUnspecified() {
+ return "", nil
+ }
+ if ipLocator == nil {
+ ipLocator = NewGaodeIPLocator(lbsServiceKey)
+ }
+ loc, err := ipLocator.GetIPLocation(clientIP)
+ if err != nil {
+ return "", fmt.Errorf("failed to get ip location, %w", err)
+ }
+
+ city := loc.City
+ return getCdnDomainByCity(city), nil
+}
+
+func getCdnDomainByCity(city string) string {
+ if cdnDomain, ok := cityToCdnDomain[city]; ok {
+ return cdnDomain
+ }
+ return ""
+}
+
+
+
package geo
+
+import "opencsg.com/csghub-server/common/config"
+
+var ipLocator IPLocator
+var cityToCdnDomain map[string]string
+var lbsServiceKey string
+
+func Config(config *config.Config) {
+ cityToCdnDomain = config.CityToCdnDomain
+ lbsServiceKey = config.LBSServiceKey
+}
+
+// SetIPLocator changes the default ip locator
+func SetIPLocator(locator IPLocator) {
+ ipLocator = locator
+}
+
+
+
package geo
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+)
+
+type IPLocator interface {
+ GetIPLocation(ip string) (*IPLocation, error)
+}
+
+type IPLocation struct {
+ Nation string `json:"nation"`
+ Province string `json:"province"`
+ City string `json:"city"`
+ District string `json:"district"`
+ ADCode string `json:"adcode"`
+}
+
+type gaodeIPLocator struct {
+ host string
+ api string
+
+ Key string
+}
+
+func NewGaodeIPLocator(key string) IPLocator {
+ return &gaodeIPLocator{
+ host: "https://restapi.amap.com",
+ api: "/v3/ip",
+ Key: key,
+ }
+}
+
+func (g *gaodeIPLocator) GetIPLocation(ip string) (*IPLocation, error) {
+ //see gaode api doc: https://lbs.amap.com/api/webservice/guide/api/ipconfig/
+ url := fmt.Sprintf("%s%s?ip=%s&key=%s", g.host, g.api, ip, g.Key)
+ resp, err := http.Get(url)
+ if err != nil {
+ return nil, fmt.Errorf("failed to call gaode ip location api: %w", err)
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read gaode ip location api response: %w", err)
+ }
+ var gaodeIPLocationResponse gaodeIPLocationResponse
+ err = json.Unmarshal(body, &gaodeIPLocationResponse)
+ if err != nil {
+ return nil, fmt.Errorf("failed to unmarshal gaode ip location api response: %w", err)
+ }
+ if gaodeIPLocationResponse.Status != "1" {
+ return nil, fmt.Errorf("gaode ip location api status is not 1, message: %s", gaodeIPLocationResponse.Info)
+ }
+ return &IPLocation{
+ Province: gaodeIPLocationResponse.Province,
+ City: gaodeIPLocationResponse.City,
+ ADCode: gaodeIPLocationResponse.ADCode,
+ }, nil
+}
+
+type gaodeIPLocationResponse struct {
+ Status string `json:"status"`
+ Info string `json:"info"`
+ InfoCode string `json:"infocode"`
+ Province string `json:"province"`
+ City string `json:"city"`
+ ADCode string `json:"adcode"`
+}
+
+
+
package git
+
+import (
+ "errors"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/gitserver/gitaly"
+ "opencsg.com/csghub-server/builder/git/gitserver/gitea"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func NewGitServer(config *config.Config) (gitserver.GitServer, error) {
+ if config.GitServer.Type == types.GitServerTypeGitea {
+ gitServer, err := gitea.NewClient(config)
+ return gitServer, err
+ } else if config.GitServer.Type == types.GitServerTypeGitaly {
+ gitServer, err := gitaly.NewClient(config)
+ return gitServer, err
+ }
+
+ return nil, errors.New("undefined git server type")
+}
+
+
+
package gitaly
+
+import (
+ "strings"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) CreateUserToken(req *types.CreateUserTokenRequest) (token *database.AccessToken, err error) {
+ token = &database.AccessToken{
+ Name: req.TokenName,
+ Permission: req.Permission,
+ Application: req.Application,
+ ExpiredAt: req.ExpiredAt,
+ Token: strings.ReplaceAll(uuid.NewString(), "-", ""),
+ }
+ return
+}
+
+func (c *Client) DeleteUserToken(req *types.DeleteUserTokenRequest) (err error) {
+ return
+}
+
+
+
package gitaly
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "path/filepath"
+
+ "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) GetRepoBranches(ctx context.Context, req gitserver.GetBranchesReq) ([]types.Branch, error) {
+ var branches []types.Branch
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ branchesReq := &gitalypb.FindAllBranchesRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ }
+ stream, err := c.refClient.FindAllBranches(ctx, branchesReq)
+ if err != nil {
+ return nil, err
+ }
+ for {
+ resp, err := stream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if resp != nil {
+ for _, branch := range resp.Branches {
+ branches = append(branches, types.Branch{
+ Name: filepath.Base(string(branch.Name)),
+ Message: string(branch.Target.Subject),
+ Commit: types.RepoBranchCommit{
+ ID: branch.Target.Id,
+ },
+ })
+ }
+ }
+ }
+
+ return branches, nil
+}
+
+
+
package gitaly
+
+import (
+ "context"
+
+ log "github.com/sirupsen/logrus"
+ gitalyauth "gitlab.com/gitlab-org/gitaly/v16/auth"
+ gitalyclient "gitlab.com/gitlab-org/gitaly/v16/client"
+ gitalypb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/config"
+)
+
+var _ gitserver.GitServer = (*Client)(nil)
+
+type Client struct {
+ config *config.Config
+ sidechannelRegistry *gitalyclient.SidechannelRegistry
+ repoClient gitalypb.RepositoryServiceClient
+ commitClient gitalypb.CommitServiceClient
+ blobClient gitalypb.BlobServiceClient
+ refClient gitalypb.RefServiceClient
+ diffClient gitalypb.DiffServiceClient
+ operationClient gitalypb.OperationServiceClient
+ smartHttpClient gitalypb.SmartHTTPServiceClient
+ remoteClient gitalypb.RemoteServiceClient
+}
+
+func NewClient(config *config.Config) (*Client, error) {
+ var sidechannelRegistry *gitalyclient.SidechannelRegistry
+ accessLogger := log.New()
+ accessLogger.SetLevel(log.InfoLevel)
+ sidechannelRegistry = gitalyclient.NewSidechannelRegistry(log.NewEntry(accessLogger))
+ connOpts := append(gitalyclient.DefaultDialOpts,
+ grpc.WithPerRPCCredentials(gitalyauth.RPCCredentialsV2(config.GitalyServer.Token)),
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ gitalyclient.WithGitalyDNSResolver(gitalyclient.DefaultDNSResolverBuilderConfig()),
+ )
+
+ conn, connErr := gitalyclient.DialSidechannel(context.Background(), config.GitalyServer.Address, sidechannelRegistry, connOpts)
+ repoClient := gitalypb.NewRepositoryServiceClient(conn)
+ commitClient := gitalypb.NewCommitServiceClient(conn)
+ blobClient := gitalypb.NewBlobServiceClient(conn)
+ refClient := gitalypb.NewRefServiceClient(conn)
+ diffClient := gitalypb.NewDiffServiceClient(conn)
+ operationClient := gitalypb.NewOperationServiceClient(conn)
+ smartHttpClient := gitalypb.NewSmartHTTPServiceClient(conn)
+ remoteClient := gitalypb.NewRemoteServiceClient(conn)
+
+ if connErr != nil {
+ return nil, connErr
+ }
+
+ return &Client{
+ config: config,
+ sidechannelRegistry: sidechannelRegistry,
+ repoClient: repoClient,
+ commitClient: commitClient,
+ blobClient: blobClient,
+ refClient: refClient,
+ diffClient: diffClient,
+ operationClient: operationClient,
+ smartHttpClient: smartHttpClient,
+ remoteClient: remoteClient,
+ }, nil
+}
+
+
+
package gitaly
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "time"
+
+ "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+)
+
+const SHA1EmptyTreeID = "4b825dc642cb6eb9a060e54bf8d69288fbee4904"
+
+func (c *Client) GetRepoCommits(ctx context.Context, req gitserver.GetRepoCommitsReq) ([]types.Commit, *types.RepoPageOpts, error) {
+ var commits []types.Commit
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ commitsReq := &gitalypb.FindCommitsRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Revision: []byte(req.Ref),
+ Limit: int32(req.Per),
+ Offset: int32(req.Per * (req.Page - 1)),
+ }
+ stream, err := c.commitClient.FindCommits(ctx, commitsReq)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for {
+ resp, err := stream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, nil, err
+ }
+ if resp != nil {
+ for _, commit := range resp.Commits {
+ commits = append(commits, types.Commit{
+ ID: string(commit.Id),
+ CommitterName: string(commit.Committer.Name),
+ CommitterEmail: string(commit.Committer.Email),
+ CommitterDate: commit.Committer.Date.AsTime().Format(time.RFC3339),
+ CreatedAt: commit.Committer.Date.AsTime().Format(time.RFC3339),
+ Message: string(commit.Subject),
+ AuthorName: string(commit.Author.Name),
+ AuthorEmail: string(commit.Author.Email),
+ AuthoredDate: commit.Author.Date.AsTime().Format(time.RFC3339),
+ })
+ }
+ }
+ }
+
+ countCommitsReq := &gitalypb.CountCommitsRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Revision: []byte(req.Ref),
+ }
+ count, err := c.commitClient.CountCommits(ctx, countCommitsReq)
+ if err != nil {
+ return nil, nil, err
+ }
+ repoPageOpts := &types.RepoPageOpts{
+ Total: int(count.Count),
+ PageCount: int(math.Ceil(float64(count.Count) / float64(req.Per))),
+ }
+
+ return commits, repoPageOpts, nil
+}
+
+func (c *Client) GetRepoLastCommit(ctx context.Context, req gitserver.GetRepoLastCommitReq) (*types.Commit, error) {
+ var commit types.Commit
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ commitReq := &gitalypb.FindCommitRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Revision: []byte(req.Ref),
+ }
+ resp, err := c.commitClient.FindCommit(ctx, commitReq)
+ if err != nil {
+ return nil, err
+ }
+ if resp != nil && resp.Commit != nil {
+ commit = types.Commit{
+ ID: string(resp.Commit.Id),
+ CommitterName: string(resp.Commit.Committer.Name),
+ CommitterEmail: string(resp.Commit.Committer.Email),
+ CommitterDate: resp.Commit.Committer.Date.AsTime().Format(time.RFC3339),
+ CreatedAt: resp.Commit.Committer.Date.AsTime().Format(time.RFC3339),
+ Message: string(resp.Commit.Subject),
+ AuthorName: string(resp.Commit.Author.Name),
+ AuthorEmail: string(resp.Commit.Author.Email),
+ AuthoredDate: resp.Commit.Author.Date.AsTime().Format(time.RFC3339),
+ }
+ }
+
+ return &commit, nil
+}
+
+func (c *Client) GetCommitDiff(ctx context.Context, req gitserver.GetRepoLastCommitReq) ([]byte, error) {
+ return nil, nil
+}
+
+func (c *Client) GetSingleCommit(ctx context.Context, req gitserver.GetRepoLastCommitReq) (*types.CommitResponse, error) {
+ var (
+ result types.CommitResponse
+ commit types.Commit
+ parents []*types.CommitMeta
+ files []string
+ diff []byte
+ additions int
+ deletions int
+ )
+
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ commitReq := &gitalypb.FindCommitRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Revision: []byte(req.Ref),
+ }
+ commitResp, err := c.commitClient.FindCommit(ctx, commitReq)
+ if err != nil {
+ return nil, err
+ }
+ if commitResp != nil && commitResp.Commit != nil {
+ commit = types.Commit{
+ ID: string(commitResp.Commit.Id),
+ CommitterName: string(commitResp.Commit.Committer.Name),
+ CommitterEmail: string(commitResp.Commit.Committer.Email),
+ CommitterDate: commitResp.Commit.Committer.Date.AsTime().Format(time.RFC3339),
+ CreatedAt: commitResp.Commit.Committer.Date.AsTime().Format(time.RFC3339),
+ Message: string(commitResp.Commit.Subject),
+ AuthorName: string(commitResp.Commit.Author.Name),
+ AuthorEmail: string(commitResp.Commit.Author.Email),
+ AuthoredDate: commitResp.Commit.Author.Date.AsTime().Format(time.RFC3339),
+ }
+ for _, id := range commitResp.Commit.ParentIds {
+ parents = append(parents, &types.CommitMeta{
+ SHA: id,
+ })
+ }
+ } else {
+ return nil, errors.New("commit not found")
+ }
+ result = types.CommitResponse{
+ Commit: &commit,
+ Parents: parents,
+ }
+
+ diffCtx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ filesReq := &gitalypb.DiffStatsRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ RightCommitId: req.Ref,
+ }
+ if len(parents) > 0 {
+ filesReq.LeftCommitId = parents[0].SHA
+ } else {
+ filesReq.LeftCommitId = SHA1EmptyTreeID
+ }
+ fileStream, err := c.diffClient.DiffStats(ctx, filesReq)
+ if err != nil {
+ return nil, err
+ }
+ for {
+ data, err := fileStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if data != nil {
+ for _, stat := range data.Stats {
+ files = append(files, string(stat.Path))
+ additions += int(stat.Additions)
+ deletions += int(stat.Deletions)
+ }
+ }
+ }
+ result.Files = files
+ result.Stats = &types.CommitStats{
+ Additions: additions,
+ Deletions: deletions,
+ Total: additions + deletions,
+ }
+
+ diffReq := &gitalypb.RawDiffRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ RightCommitId: req.Ref,
+ }
+ if len(parents) > 0 {
+ diffReq.LeftCommitId = parents[0].SHA
+ } else {
+ diffReq.LeftCommitId = SHA1EmptyTreeID
+ }
+ diffStream, err := c.diffClient.RawDiff(diffCtx, diffReq)
+ if err != nil {
+ return nil, err
+ }
+ for {
+ data, err := diffStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if data != nil {
+ diff = data.Data
+ }
+ }
+ result.Diff = diff
+
+ return &result, nil
+}
+
+func (c *Client) GetDiffBetweenTwoCommits(ctx context.Context, req gitserver.GetDiffBetweenTwoCommitsReq) (*types.GiteaCallbackPushReq, error) {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ callback := &types.GiteaCallbackPushReq{
+ Ref: req.Ref,
+ Repository: types.GiteaCallbackPushReq_Repository{
+ FullName: fmt.Sprintf("%s_%s/%s", repoType, req.Namespace, req.Name),
+ Private: req.Private,
+ },
+ }
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ }
+ treeReq := &gitalypb.FindChangedPathsRequest_Request_TreeRequest{
+ RightTreeRevision: req.RightCommitId,
+ }
+ if req.LeftCommitId != "0000000000000000000000000000000000000000" {
+ treeReq.LeftTreeRevision = req.LeftCommitId
+ } else {
+ treeReq.LeftTreeRevision = SHA1EmptyTreeID
+ }
+
+ commitReq := &gitalypb.FindChangedPathsRequest{
+ Repository: repository,
+ MergeCommitDiffMode: gitalypb.FindChangedPathsRequest_MERGE_COMMIT_DIFF_MODE_UNSPECIFIED,
+ Requests: []*gitalypb.FindChangedPathsRequest_Request{
+ {
+ Type: &gitalypb.FindChangedPathsRequest_Request_TreeRequest_{
+ TreeRequest: treeReq,
+ },
+ },
+ },
+ }
+ commitResp, err := c.diffClient.FindChangedPaths(ctx, commitReq)
+ if err != nil {
+ return nil, err
+ }
+
+ if commitResp != nil {
+ var commits []types.GiteaCallbackPushReq_Commit
+ for {
+ data, err := commitResp.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if data != nil {
+ var (
+ modified []string
+ added []string
+ deleted []string
+ )
+ for _, path := range data.Paths {
+ if path.Status == gitalypb.ChangedPaths_ADDED {
+ added = append(added, string(path.Path))
+ } else if path.Status == gitalypb.ChangedPaths_DELETED {
+ deleted = append(deleted, string(path.Path))
+ } else if path.Status == gitalypb.ChangedPaths_MODIFIED {
+ modified = append(modified, string(path.Path))
+ }
+ }
+ commits = append(commits, types.GiteaCallbackPushReq_Commit{
+ Added: added,
+ Removed: deleted,
+ Modified: modified,
+ })
+ }
+ }
+ callback.Commits = commits
+ }
+
+ findCommitReq := &gitalypb.FindCommitRequest{
+ Repository: repository,
+ Revision: []byte(req.RightCommitId),
+ }
+
+ findCommitResp, err := c.commitClient.FindCommit(ctx, findCommitReq)
+ if err != nil {
+ return nil, err
+ }
+
+ if findCommitResp != nil {
+ callback.HeadCommit = types.GiteaCallbackPushReq_HeadCommit{
+ Timestamp: findCommitResp.Commit.Committer.Date.AsTime().Format(time.RFC3339),
+ Message: string(findCommitResp.Commit.Subject),
+ LastModifyTime: findCommitResp.Commit.Committer.Date.AsTime(),
+ }
+ }
+ return callback, nil
+}
+
+
+
package gitaly
+
+import "strings"
+
+func BuildRelativePath(repoType, namespace, name string) string {
+ return strings.ToLower(repoType + "_" + namespace + "/" + name + ".git")
+}
+
+
+
package gitaly
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "time"
+
+ "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/status"
+ "google.golang.org/protobuf/types/known/timestamppb"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+)
+
+const (
+ LFSPrefix = "version https://git-lfs.github.com/spec/v1"
+ NonLFSFileSizeLimit = 10485760
+ GitAttributesFileName = ".gitattributes"
+)
+
+func (c *Client) GetRepoFileRaw(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (string, error) {
+ var data []byte
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ }
+
+ req.Path = strings.TrimPrefix(req.Path, "/")
+
+ if req.Ref == "" {
+ req.Ref = "main"
+ }
+
+ treeEntriesReq := &gitalypb.TreeEntryRequest{
+ Repository: repository,
+ Revision: []byte(req.Ref),
+ Path: []byte(req.Path),
+ MaxSize: req.MaxFileSize,
+ }
+ treeEntriesStream, err := c.commitClient.TreeEntry(ctx, treeEntriesReq)
+ if err != nil {
+ return "", err
+ }
+
+ for {
+ treeEntriesResp, err := treeEntriesStream.Recv()
+ if err != nil {
+ grpcStatus, ok := status.FromError(err)
+ if ok && grpcStatus.Code() == codes.FailedPrecondition && strings.Contains(grpcStatus.Message(), "bigger than the maximum allowed size") {
+ return "", gitserver.ErrFileTooLarge
+ }
+ if err == io.EOF {
+ break
+ }
+ return "", err
+ }
+ if treeEntriesResp != nil {
+ data = append(data, treeEntriesResp.Data...)
+ }
+ }
+
+ return string(data), nil
+}
+
+func (c *Client) GetRepoFileReader(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (io.ReadCloser, int64, error) {
+ var size int64
+ sizeChan := make(chan int64, 1)
+ pr, pw := io.Pipe()
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ // if we add cancel function here, it will break the download stream
+ // ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ // defer cancel()
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ }
+
+ treeEntriesReq := &gitalypb.TreeEntryRequest{
+ Repository: repository,
+ Revision: []byte(req.Ref),
+ Path: []byte(req.Path),
+ }
+
+ treeEntriesStream, err := c.commitClient.TreeEntry(ctx, treeEntriesReq)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ go func() {
+ defer pw.Close()
+
+ for {
+ treeEntriesResp, err := treeEntriesStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ pw.CloseWithError(fmt.Errorf("failed to receive data: %v", err))
+ return
+ }
+
+ if treeEntriesResp.Size != 0 {
+ sizeChan <- treeEntriesResp.Size
+ }
+
+ if len(treeEntriesResp.Data) > 0 {
+ if _, err := pw.Write(treeEntriesResp.Data); err != nil {
+ pw.CloseWithError(fmt.Errorf("failed to write data to pipe: %v", err))
+ return
+ }
+ }
+ }
+ }()
+ size = <-sizeChan
+
+ return pr, size, nil
+}
+
+func (c *Client) GetRepoLfsFileRaw(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (io.ReadCloser, error) {
+ return nil, nil
+}
+
+// GetRepoFileContents returns the file basic info and content of a file in a repository.
+//
+// If the file is larger than the maximum allowed size, it will return the
+// file basic info, but not content, and an ErrFileTooLarge error.
+func (c *Client) GetRepoFileContents(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (*types.File, error) {
+ req.File = true
+ files, err := c.GetRepoFileTree(ctx, req)
+ if err != nil {
+ return nil, err
+ }
+ file := files[0]
+ content, err := c.GetRepoFileRaw(ctx, req)
+ if err != nil {
+ if errors.Is(err, gitserver.ErrFileTooLarge) {
+ // return file basic info, but not content
+ return file, err
+ }
+ return nil, err
+ }
+ file.Content = base64.StdEncoding.EncodeToString([]byte(content))
+
+ return file, nil
+}
+
+func (c *Client) CreateRepoFile(req *types.CreateFileReq) (err error) {
+ ctx := context.Background()
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ if req.NewBranch == "" {
+ req.NewBranch = req.Branch
+ }
+ conn, err := grpc.NewClient(
+ c.config.GitalyServer.Address,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ )
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ userCommitFilesClient, err := c.operationClient.UserCommitFiles(ctx)
+ if err != nil {
+ return err
+ }
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ GlRepository: filepath.Join(repoType, req.Namespace, req.Name),
+ }
+
+ header := &gitalypb.UserCommitFilesRequestHeader{
+ Repository: repository,
+ User: &gitalypb.User{
+ GlId: "user-1",
+ Name: []byte(req.Name),
+ GlUsername: req.Username,
+ Email: []byte(req.Email),
+ },
+ BranchName: []byte(req.NewBranch),
+ CommitMessage: []byte(req.Message),
+ CommitAuthorName: []byte(req.Name),
+ CommitAuthorEmail: []byte(req.Email),
+ StartRepository: repository,
+ Timestamp: timestamppb.New(time.Now()),
+ }
+
+ if req.Branch != "" {
+ header.StartBranchName = []byte(req.Branch)
+ }
+ actions := []*gitalypb.UserCommitFilesRequest{
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Header{
+ Header: header,
+ },
+ },
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Action{
+ Action: &gitalypb.UserCommitFilesAction{
+ UserCommitFilesActionPayload: &gitalypb.UserCommitFilesAction_Header{
+ Header: &gitalypb.UserCommitFilesActionHeader{
+ Action: gitalypb.UserCommitFilesActionHeader_CREATE,
+ Base64Content: true,
+ FilePath: []byte(req.FilePath),
+ },
+ },
+ },
+ },
+ },
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Action{
+ Action: &gitalypb.UserCommitFilesAction{
+ UserCommitFilesActionPayload: &gitalypb.UserCommitFilesAction_Content{
+ Content: []byte(req.Content),
+ },
+ },
+ },
+ },
+ }
+ err = userCommitFilesClient.Send(actions[0])
+ if err != nil {
+ return err
+ }
+ err = userCommitFilesClient.Send(actions[1])
+ if err != nil {
+ return err
+ }
+ err = userCommitFilesClient.Send(actions[2])
+ if err != nil {
+ return err
+ }
+ _, err = userCommitFilesClient.CloseAndRecv()
+ if err != nil {
+ return err
+ }
+
+ return err
+}
+
+func (c *Client) UpdateRepoFile(req *types.UpdateFileReq) (err error) {
+ ctx := context.Background()
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ conn, err := grpc.NewClient(
+ c.config.GitalyServer.Address,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ )
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ userCommitFilesClient, err := c.operationClient.UserCommitFiles(ctx)
+ if err != nil {
+ return err
+ }
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ GlRepository: filepath.Join(repoType, req.Namespace, req.Name),
+ }
+ actions := []*gitalypb.UserCommitFilesRequest{
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Header{
+ Header: &gitalypb.UserCommitFilesRequestHeader{
+ Repository: repository,
+ User: &gitalypb.User{
+ GlId: "user-1",
+ Name: []byte(req.Name),
+ GlUsername: req.Username,
+ Email: []byte(req.Email),
+ },
+ BranchName: []byte(req.Branch),
+ CommitMessage: []byte(req.Message),
+ CommitAuthorName: []byte(req.Name),
+ CommitAuthorEmail: []byte(req.Email),
+ StartBranchName: []byte(req.Branch),
+ StartRepository: repository,
+ Timestamp: timestamppb.New(time.Now()),
+ },
+ },
+ },
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Action{
+ Action: &gitalypb.UserCommitFilesAction{
+ UserCommitFilesActionPayload: &gitalypb.UserCommitFilesAction_Header{
+ Header: &gitalypb.UserCommitFilesActionHeader{
+ Action: gitalypb.UserCommitFilesActionHeader_UPDATE,
+ Base64Content: true,
+ FilePath: []byte(req.FilePath),
+ },
+ },
+ },
+ },
+ },
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Action{
+ Action: &gitalypb.UserCommitFilesAction{
+ UserCommitFilesActionPayload: &gitalypb.UserCommitFilesAction_Content{
+ Content: []byte(req.Content),
+ },
+ },
+ },
+ },
+ }
+ err = userCommitFilesClient.Send(actions[0])
+ if err != nil {
+ return err
+ }
+ err = userCommitFilesClient.Send(actions[1])
+ if err != nil {
+ return err
+ }
+ err = userCommitFilesClient.Send(actions[2])
+ if err != nil {
+ return err
+ }
+ _, err = userCommitFilesClient.CloseAndRecv()
+ if err != nil {
+ return err
+ }
+
+ return err
+}
+
+func (c *Client) DeleteRepoFile(req *types.DeleteFileReq) (err error) {
+ ctx := context.Background()
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ conn, err := grpc.NewClient(
+ c.config.GitalyServer.Address,
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ )
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ userCommitFilesClient, err := c.operationClient.UserCommitFiles(ctx)
+ if err != nil {
+ return err
+ }
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ GlRepository: filepath.Join(repoType, req.Namespace, req.Name),
+ }
+ actions := []*gitalypb.UserCommitFilesRequest{
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Header{
+ Header: &gitalypb.UserCommitFilesRequestHeader{
+ Repository: repository,
+ User: &gitalypb.User{
+ GlId: "user-1",
+ Name: []byte(req.Name),
+ GlUsername: req.Username,
+ Email: []byte(req.Email),
+ },
+ BranchName: []byte(req.Branch),
+ CommitMessage: []byte(req.Message),
+ CommitAuthorName: []byte(req.Name),
+ CommitAuthorEmail: []byte(req.Email),
+ StartBranchName: []byte(req.Branch),
+ StartRepository: repository,
+ Timestamp: timestamppb.New(time.Now()),
+ },
+ },
+ },
+ {
+ UserCommitFilesRequestPayload: &gitalypb.UserCommitFilesRequest_Action{
+ Action: &gitalypb.UserCommitFilesAction{
+ UserCommitFilesActionPayload: &gitalypb.UserCommitFilesAction_Header{
+ Header: &gitalypb.UserCommitFilesActionHeader{
+ Action: gitalypb.UserCommitFilesActionHeader_DELETE,
+ Base64Content: true,
+ FilePath: []byte(req.FilePath),
+ },
+ },
+ },
+ },
+ },
+ }
+ err = userCommitFilesClient.Send(actions[0])
+ if err != nil {
+ return err
+ }
+ err = userCommitFilesClient.Send(actions[1])
+ if err != nil {
+ return err
+ }
+ _, err = userCommitFilesClient.CloseAndRecv()
+ if err != nil {
+ return err
+ }
+
+ return err
+}
+
+func (c *Client) GetRepoFileTree(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error) {
+ var files []*types.File
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, time.Second*3)
+ defer cancel()
+
+ req.Path = strings.TrimPrefix(req.Path, "/")
+
+ if !req.File {
+ req.Path = req.Path + "/"
+ }
+
+ if req.Ref == "" {
+ req.Ref = "main"
+ }
+ repository := &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ }
+
+ var revision_paths []*gitalypb.GetBlobsRequest_RevisionPath
+
+ // Get last commit
+ pathCommitMap := make(map[string]*gitalypb.GitCommit)
+ gitalyReq := &gitalypb.ListLastCommitsForTreeRequest{
+ Repository: repository,
+ Revision: req.Ref,
+ Path: []byte(req.Path),
+ Limit: 1000,
+ LiteralPathspec: true,
+ }
+ commitStream, err := c.commitClient.ListLastCommitsForTree(ctx, gitalyReq)
+ if err != nil {
+ return nil, err
+ }
+ for {
+ commitResp, err := commitStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ }
+ if commitResp == nil {
+ return nil, errors.New("bad request")
+ }
+ commits := commitResp.Commits
+ if len(commits) > 0 {
+ for _, c := range commits {
+ pathCommitMap[string(c.PathBytes)] = c.Commit
+ revision_paths = append(revision_paths, &gitalypb.GetBlobsRequest_RevisionPath{
+ Revision: req.Ref,
+ Path: c.PathBytes,
+ })
+ }
+ }
+ }
+
+ // Get blobs with file size
+ listBlobsReq := &gitalypb.GetBlobsRequest{
+ Repository: repository,
+ RevisionPaths: revision_paths,
+ Limit: 1024,
+ }
+
+ listBlobsStream, err := c.blobClient.GetBlobs(ctx, listBlobsReq)
+ if err != nil {
+ return nil, err
+ }
+ for {
+ listBlobResp, err := listBlobsStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if listBlobResp != nil {
+ var (
+ fileType string
+ fileSize int64
+ isLfs bool
+ lfsPointerSize int
+ LfsRelativePath string
+ )
+ filename := filepath.Base(string(listBlobResp.Path))
+ if listBlobResp.Type == gitalypb.ObjectType_BLOB {
+ fileType = "file"
+ } else {
+ fileType = "dir"
+ }
+ fileSize = listBlobResp.Size
+ if listBlobResp.Size <= 1024 {
+ p, _ := ReadPointerFromBuffer(listBlobResp.Data)
+ if p.Valid() {
+ fileSize = p.Size
+ isLfs = true
+ LfsRelativePath = p.RelativePath()
+ lfsPointerSize = int(listBlobResp.Size)
+ }
+ }
+ file := &types.File{
+ Name: filename,
+ Type: fileType,
+ Size: fileSize,
+ Lfs: isLfs,
+ Path: string(listBlobResp.Path),
+ Mode: strconv.Itoa(int(listBlobResp.Mode)),
+ SHA: listBlobResp.Oid,
+ LfsPointerSize: lfsPointerSize,
+ LfsRelativePath: LfsRelativePath,
+ }
+ commit := pathCommitMap[string(listBlobResp.Path)]
+ if commit != nil {
+ file.Commit = types.Commit{
+ ID: commit.Id,
+ CommitterName: string(commit.Committer.Name),
+ CommitterEmail: string(commit.Committer.Email),
+ CommitterDate: commit.Committer.Date.AsTime().Format(time.RFC3339),
+ CreatedAt: commit.Committer.Date.AsTime().Format(time.RFC3339),
+ Message: string(commit.Subject),
+ AuthorName: string(commit.Author.Name),
+ AuthorEmail: string(commit.Author.Email),
+ AuthoredDate: commit.Author.Date.AsTime().Format(time.RFC3339),
+ }
+ file.LastCommitSHA = commit.Id
+ }
+
+ files = append(files, file)
+ }
+ }
+
+ return files, nil
+}
+
+func (c *Client) GetRepoAllFiles(ctx context.Context, req gitserver.GetRepoAllFilesReq) ([]*types.File, error) {
+ var files []*types.File
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, time.Second*3)
+ defer cancel()
+
+ allFilesReq := &gitalypb.ListFilesRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Revision: []byte(req.Ref),
+ }
+
+ allFilesStream, err := c.commitClient.ListFiles(ctx, allFilesReq)
+ if err != nil {
+ return nil, err
+ }
+
+ for {
+ allFilesResp, err := allFilesStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if allFilesResp != nil {
+ for _, path := range allFilesResp.Paths {
+ files = append(files, &types.File{
+ Name: filepath.Base(string(path)),
+ Path: string(path),
+ })
+ }
+
+ }
+ }
+ return files, nil
+}
+
+func (c *Client) GetRepoAllLfsPointers(ctx context.Context, req gitserver.GetRepoAllFilesReq) ([]*types.LFSPointer, error) {
+ var pointers []*types.LFSPointer
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, time.Second*3)
+ defer cancel()
+
+ allPointersReq := &gitalypb.ListAllLFSPointersRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ }
+
+ allPointersStream, err := c.blobClient.ListAllLFSPointers(ctx, allPointersReq)
+ if err != nil {
+ return nil, err
+ }
+ for {
+ allPointersResp, err := allPointersStream.Recv()
+ if err != nil {
+ if err == io.EOF {
+ break
+ }
+ return nil, err
+ }
+ if allPointersResp != nil {
+ for _, pointer := range allPointersResp.LfsPointers {
+ var p types.Pointer
+ p, _ = ReadPointerFromBuffer(pointer.Data)
+ pointers = append(pointers, &types.LFSPointer{
+ Oid: pointer.Oid,
+ Size: pointer.Size,
+ FileOid: string(p.Oid),
+ FileSize: p.Size,
+ Data: string(pointer.Data),
+ })
+ }
+ }
+ }
+ return pointers, nil
+}
+
+
+
package gitaly
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "log/slog"
+ "strconv"
+
+ gitalyclient "gitlab.com/gitlab-org/gitaly/v16/client"
+ gitalypb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "gitlab.com/gitlab-org/gitaly/v16/streamio"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+)
+
+// var (
+// uploadPackTimeout = 10 * time.Minute
+// )
+
+func (c *Client) InfoRefsResponse(ctx context.Context, req gitserver.InfoRefsReq) (io.Reader, error) {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+
+ rpcRequest := &gitalypb.InfoRefsRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ GitProtocol: req.GitProtocol,
+ }
+
+ switch req.Rpc {
+ case "git-upload-pack":
+ stream, err := c.smartHttpClient.InfoRefsUploadPack(ctx, rpcRequest)
+ return infoRefsReader(stream), err
+ case "git-receive-pack":
+ stream, err := c.smartHttpClient.InfoRefsReceivePack(ctx, rpcRequest)
+ return infoRefsReader(stream), err
+ default:
+ return nil, fmt.Errorf("InfoRefsResponseWriterTo: Unsupported RPC: %q", req.Rpc)
+ }
+}
+
+func infoRefsReader(stream infoRefsClient) io.Reader {
+ return streamio.NewReader(func() ([]byte, error) {
+ resp, err := stream.Recv()
+ return resp.GetData(), err
+ })
+}
+
+type infoRefsClient interface {
+ Recv() (*gitalypb.InfoRefsResponse, error)
+}
+
+func (c *Client) UploadPack(ctx context.Context, req gitserver.UploadPackReq) error {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, waiter := c.sidechannelRegistry.Register(ctx, func(conn gitalyclient.SidechannelConn) error {
+ if _, err := io.Copy(conn, req.Request.Body); err != nil {
+ return fmt.Errorf("copy request body: %w", err)
+ }
+
+ if err := conn.CloseWrite(); err != nil {
+ return fmt.Errorf("close request body: %w", err)
+ }
+
+ if _, err := io.Copy(req.Writer, conn); err != nil {
+ return fmt.Errorf("copy response body: %w", err)
+ }
+
+ return nil
+ })
+ defer waiter.Close()
+
+ rpcRequest := &gitalypb.PostUploadPackWithSidechannelRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ GitProtocol: req.GitProtocol,
+ }
+
+ _, err := c.smartHttpClient.PostUploadPackWithSidechannel(ctx, rpcRequest)
+ if err != nil {
+ return fmt.Errorf("PostUploadPackWithSidechannel: %w", err)
+ }
+
+ if err = waiter.Close(); err != nil {
+ return fmt.Errorf("close sidechannel waiter: %w", err)
+ }
+
+ return nil
+}
+
+func (c *Client) ReceivePack(ctx context.Context, req gitserver.ReceivePackReq) error {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ stream, err := c.smartHttpClient.PostReceivePack(ctx)
+ if err != nil {
+ return err
+ }
+ glRepository := fmt.Sprintf("%s/%s/%s", req.RepoType, req.Namespace, req.Name)
+
+ rpcRequest := &gitalypb.PostReceivePackRequest{
+ Repository: &gitalypb.Repository{
+ GlRepository: glRepository,
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ GlId: fmt.Sprintf("user-%s", strconv.FormatInt(req.UserId, 10)),
+ GlUsername: req.Username,
+ GlRepository: glRepository,
+ GitProtocol: req.GitProtocol,
+ }
+
+ if err := stream.Send(rpcRequest); err != nil {
+ return fmt.Errorf("initial request: %v", err)
+ }
+
+ numStreams := 2
+ errC := make(chan error, numStreams)
+
+ go func() {
+ rr := streamio.NewReader(func() ([]byte, error) {
+ response, err := stream.Recv()
+ return response.GetData(), err
+ })
+ _, err := io.Copy(req.Writer, rr)
+ errC <- err
+ }()
+
+ go func() {
+ sw := streamio.NewWriter(func(data []byte) error {
+ return stream.Send(&gitalypb.PostReceivePackRequest{Data: data})
+ })
+ _, err := io.Copy(sw, req.Request.Body)
+ errClose := stream.CloseSend()
+ if errClose != nil {
+ slog.Error("closeSend stream failed", slog.Any("error", errClose))
+ }
+ errC <- err
+ }()
+
+ for i := 0; i < numStreams; i++ {
+ if err := <-errC; err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+
+
package gitaly
+
+import (
+ "errors"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+const (
+ // MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
+ // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+ MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+
+ // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
+ MetaFileOidPrefix = "oid sha256:"
+)
+
+var (
+ // ErrMissingPrefix occurs if the content lacks the LFS prefix
+ ErrMissingPrefix = errors.New("content lacks the LFS prefix")
+
+ // ErrInvalidStructure occurs if the content has an invalid structure
+ ErrInvalidStructure = errors.New("content has an invalid structure")
+
+ // ErrInvalidOIDFormat occurs if the oid has an invalid format
+ ErrInvalidOIDFormat = errors.New("OID has an invalid format")
+
+ oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
+)
+
+// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
+func ReadPointerFromBuffer(buf []byte) (types.Pointer, error) {
+ var p types.Pointer
+
+ headString := string(buf)
+ if !strings.HasPrefix(headString, MetaFileIdentifier) {
+ return p, ErrMissingPrefix
+ }
+
+ splitLines := strings.Split(headString, "\n")
+ if len(splitLines) < 3 {
+ return p, ErrInvalidStructure
+ }
+
+ oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
+ if len(oid) != 64 || !oidPattern.MatchString(oid) {
+ return p, ErrInvalidOIDFormat
+ }
+ size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+ if err != nil {
+ return p, err
+ }
+
+ p.Oid = oid
+ p.Size = size
+
+ return p, nil
+}
+
+
+
package gitaly
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "time"
+
+ "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+)
+
+func (c *Client) CreateMirrorRepo(ctx context.Context, req gitserver.CreateMirrorRepoReq) (int64, error) {
+ var (
+ remoteCheckReq *gitalypb.FindRemoteRepositoryRequest
+ authorHeader string
+ err error
+ )
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ if req.MirrorToken == "" {
+ remoteCheckReq = &gitalypb.FindRemoteRepositoryRequest{
+ Remote: req.CloneUrl,
+ StorageName: c.config.GitalyServer.Storge,
+ }
+
+ resp, err := c.remoteClient.FindRemoteRepository(ctx, remoteCheckReq)
+ if err != nil {
+ return 0, err
+ }
+ if !resp.Exists {
+ return 0, fmt.Errorf("invalid clone url")
+ }
+ }
+
+ gitalyReq := &gitalypb.CreateRepositoryFromURLRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Url: req.CloneUrl,
+ Mirror: true,
+ }
+
+ if req.Username != "" && req.AccessToken != "" {
+ authorHeader = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", req.Username, req.AccessToken)))
+ }
+
+ if req.MirrorToken != "" {
+ gitalyReq.HttpAuthorizationHeader = fmt.Sprintf("X-OPENCSG-Sync-Token%s", req.MirrorToken)
+ } else if authorHeader != "" {
+ gitalyReq.HttpAuthorizationHeader = authorHeader
+ } else {
+ gitalyReq.HttpAuthorizationHeader = ""
+ }
+
+ _, err = c.repoClient.CreateRepositoryFromURL(ctx, gitalyReq)
+ if err != nil {
+ return 0, err
+ }
+ return 0, nil
+}
+
+func (c *Client) CreateMirrorForExistsRepo(ctx context.Context, req gitserver.CreateMirrorRepoReq) error {
+ var authorHeader string
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ fetchRemoteReq := &gitalypb.FetchRemoteRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Force: true,
+ NoPrune: true,
+ RemoteParams: &gitalypb.Remote{
+ Url: req.CloneUrl,
+ },
+ }
+
+ if req.Username != "" && req.AccessToken != "" {
+ authorHeader = base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", req.Username, req.AccessToken)))
+ }
+
+ if authorHeader != "" {
+ fetchRemoteReq.RemoteParams.HttpAuthorizationHeader = authorHeader
+ }
+
+ _, err := c.repoClient.FetchRemote(ctx, fetchRemoteReq)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *Client) GetMirrorTaskInfo(ctx context.Context, taskId int64) (*gitserver.MirrorTaskInfo, error) {
+ return nil, nil
+}
+
+func (c *Client) MirrorSync(ctx context.Context, req gitserver.MirrorSyncReq) error {
+ var authorHeader string
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+ defer cancel()
+
+ fetchRemoteReq := &gitalypb.FetchRemoteRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ Force: true,
+ NoPrune: false,
+ RemoteParams: &gitalypb.Remote{
+ Url: req.CloneUrl,
+ },
+ CheckTagsChanged: true,
+ }
+
+ if req.MirrorToken != "" {
+ fetchRemoteReq.RemoteParams.HttpAuthorizationHeader = fmt.Sprintf("X-OPENCSG-Sync-Token%s", req.MirrorToken)
+ } else if authorHeader != "" {
+ fetchRemoteReq.RemoteParams.HttpAuthorizationHeader = authorHeader
+ } else {
+ fetchRemoteReq.RemoteParams.HttpAuthorizationHeader = ""
+ }
+
+ _, err := c.repoClient.FetchRemote(ctx, fetchRemoteReq)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+
+
package gitaly
+
+import (
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+// FixOrganization recreate organization data, ignore data duplication error
+// !should only be used for online data fixing
+func (c *Client) FixOrganization(req *types.CreateOrgReq, user database.User) error {
+ return nil
+}
+
+func (c *Client) CreateOrganization(req *types.CreateOrgReq, user database.User) (org *database.Organization, err error) {
+ return &database.Organization{
+ Name: req.Name,
+ Nickname: req.Nickname,
+ Description: req.Description,
+ User: &user,
+ UserID: user.ID,
+ }, nil
+}
+
+func (c *Client) DeleteOrganization(name string) (err error) {
+ return
+}
+
+func (c *Client) UpdateOrganization(req *types.EditOrgReq, originOrg *database.Organization) (org *database.Organization, err error) {
+ return nil, nil
+}
+
+
+
package gitaly
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ gitalypb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+)
+
+const timeoutTime = 10 * time.Second
+
+func (c *Client) CreateRepo(ctx context.Context, req gitserver.CreateRepoReq) (*gitserver.CreateRepoResp, error) {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+
+ gitalyReq := &gitalypb.CreateRepositoryRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ DefaultBranch: []byte(req.DefaultBranch),
+ }
+
+ _, err := c.repoClient.CreateRepository(ctx, gitalyReq)
+ if err != nil {
+ return nil, err
+ }
+
+ return &gitserver.CreateRepoResp{
+ Username: req.Username,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Nickname: req.Nickname,
+ Description: req.Description,
+ License: req.License,
+ DefaultBranch: req.DefaultBranch,
+ RepoType: req.RepoType,
+ GitPath: strings.TrimSuffix(BuildRelativePath(repoType, req.Namespace, req.Name), ".git"),
+ Private: req.Private,
+ }, nil
+}
+
+func (c *Client) UpdateRepo(ctx context.Context, req gitserver.UpdateRepoReq) (*gitserver.CreateRepoResp, error) {
+ return nil, nil
+}
+
+func (c *Client) DeleteRepo(ctx context.Context, req gitserver.DeleteRepoReq) error {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ gitalyReq := &gitalypb.RemoveRepositoryRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ }
+ _, err := c.repoClient.RemoveRepository(ctx, gitalyReq)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (c *Client) GetRepo(ctx context.Context, req gitserver.GetRepoReq) (*gitserver.CreateRepoResp, error) {
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ ctx, cancel := context.WithTimeout(ctx, timeoutTime)
+ defer cancel()
+ gitalyReq := &gitalypb.FindDefaultBranchNameRequest{
+ Repository: &gitalypb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: BuildRelativePath(repoType, req.Namespace, req.Name),
+ },
+ }
+ resp, err := c.refClient.FindDefaultBranchName(ctx, gitalyReq)
+ if err != nil {
+ return nil, err
+ }
+
+ return &gitserver.CreateRepoResp{DefaultBranch: string(resp.Name)}, nil
+}
+
+
+
package gitaly
+
+import (
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) CreateSSHKey(req *types.CreateSSHKeyRequest) (token *database.SSHKey, err error) {
+ return
+}
+
+//List all SSH keys from gitea
+
+// func (c *Client) ListSSHKeys(username string, per, page int) (tokens []*database.SSHKey, err error) {
+// giteaSSHKeys, _, err := c.giteaClient.ListPublicKeys(
+// username,
+// gitea.ListPublicKeysOptions{
+// ListOptions: gitea.ListOptions{
+// Page: page,
+// PageSize: per,
+// },
+// },
+// )
+
+// if err != nil {
+// return
+// }
+
+// for _, giteaSSHKey := range giteaSSHKeys {
+// tokens = append(tokens, &database.SSHKey{
+// GID: int(giteaSSHKey.ID),
+// Name: giteaSSHKey.Title,
+// Content: giteaSSHKey.Key,
+// })
+// }
+
+// return
+// }
+
+func (c *Client) DeleteSSHKey(id int) (err error) {
+ return
+}
+
+
+
package gitaly
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) GetRepoTags(ctx context.Context, req gitserver.GetRepoTagsReq) (tags []*types.Tag, err error) {
+ return
+}
+
+
+
package gitaly
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) CreateUser(u gitserver.CreateUserRequest) (user *gitserver.CreateUserResponse, err error) {
+ return &gitserver.CreateUserResponse{
+ GitID: 0,
+ Password: "",
+ }, nil
+}
+
+func (c *Client) UpdateUser(u *types.UpdateUserRequest, user *database.User) (*database.User, error) {
+ return nil, nil
+}
+
+func (c *Client) UpdateUserV2(u gitserver.UpdateUserRequest) error {
+ return nil
+}
+
+// Create gitea orgs for user to store different type repositories
+func (c *Client) FixUserData(ctx context.Context, userName string) (err error) {
+ return
+}
+
+
+
package gitea
+
+import (
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) CreateUserToken(req *types.CreateUserTokenRequest) (token *database.AccessToken, err error) {
+ giteaToken, _, err := c.giteaClient.CreateAccessToken(
+ gitea.CreateAccessTokenOption{
+ Username: req.Username,
+ Name: req.TokenName,
+ Scopes: []gitea.AccessTokenScope{"write:repository"},
+ },
+ )
+
+ if err != nil {
+ return
+ }
+
+ token = &database.AccessToken{
+ GitID: giteaToken.ID,
+ Name: req.TokenName,
+ Token: giteaToken.Token,
+ }
+
+ return
+}
+
+func (c *Client) DeleteUserToken(req *types.DeleteUserTokenRequest) (err error) {
+ _, err = c.giteaClient.DeleteAccessToken(gitea.DeleteAccessTokenOption{
+ Username: req.Username,
+ Value: req.TokenName,
+ })
+ return
+}
+
+
+
package gitea
+
+import (
+ "context"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func (c *Client) GetRepoBranches(ctx context.Context, req gitserver.GetBranchesReq) ([]types.Branch, error) {
+ var branches []types.Branch
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ giteaBranches, _, err := c.giteaClient.ListRepoBranches(
+ namespace,
+ req.Name,
+ gitea.ListRepoBranchesOptions{
+ ListOptions: gitea.ListOptions{
+ PageSize: req.Per,
+ Page: req.Page,
+ },
+ },
+ )
+ for _, giteaBranch := range giteaBranches {
+ branches = append(branches, types.Branch{
+ Name: giteaBranch.Name,
+ Message: giteaBranch.Commit.Message,
+ Commit: types.RepoBranchCommit{
+ ID: giteaBranch.Commit.ID,
+ },
+ })
+ }
+ return branches, err
+}
+
+
+
package gitea
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "time"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+)
+
+var _ gitserver.GitServer = (*Client)(nil)
+
+type Client struct {
+ giteaClient *gitea.Client
+ config *config.Config
+}
+
+type Response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ Data any `json:"data"`
+}
+
+type TokenResponse struct {
+ SHA1 string `json:"sha1"`
+ Message string `json:"message"`
+}
+
+func NewClient(config *config.Config) (client *Client, err error) {
+ httpClient := &http.Client{
+ Timeout: time.Duration(config.GitServer.TimeoutSEC) * time.Second,
+ }
+ token, err := findOrCreateAccessToken(context.Background(), config)
+ if err != nil {
+ slog.Error("Failed to find or create token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+ giteaClient, err := gitea.NewClient(
+ config.GitServer.Host,
+ gitea.SetHTTPClient(httpClient),
+ gitea.SetToken(token.Token),
+ gitea.SetBasicAuth(config.GitServer.Username, config.GitServer.Password),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return &Client{giteaClient: giteaClient, config: config}, nil
+}
+
+func findOrCreateAccessToken(ctx context.Context, config *config.Config) (*database.GitServerAccessToken, error) {
+ gs := database.NewGitServerAccessTokenStore()
+ tokens, err := gs.FindByType(ctx, "git")
+ if err != nil {
+ slog.Error("Fail to get git server access token from database", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+
+ if len(tokens) == 0 {
+ access_token, err := generateAccessTokenFromGitea(config)
+ if err != nil {
+ slog.Error("Fail to create git server access token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+ gToken := &database.GitServerAccessToken{
+ Token: access_token,
+ ServerType: "git",
+ }
+
+ gToken, err = gs.Create(ctx, gToken)
+ if err != nil {
+ slog.Error("Fail to create git server access token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+
+ return gToken, nil
+ }
+ return &tokens[0], nil
+}
+
+func encodeCredentials(username, password string) string {
+ credentials := username + ":" + password
+ return base64.StdEncoding.EncodeToString([]byte(credentials))
+}
+
+func generateAccessTokenFromGitea(config *config.Config) (string, error) {
+ username := config.GitServer.Username
+ password := config.GitServer.Password
+ giteaUrl := fmt.Sprintf("%s/api/v1/users/%s/tokens", config.GitServer.Host, username)
+ authHeader := encodeCredentials(username, password)
+ data := map[string]any{
+ "name": "access_token",
+ "scopes": []string{
+ "write:user",
+ "write:admin",
+ "write:organization",
+ "write:repository",
+ },
+ }
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ slog.Error("Error encoding JSON data:", "error", err)
+ return "", err
+ }
+
+ req, err := http.NewRequest("POST", giteaUrl, bytes.NewBuffer(jsonData))
+ if err != nil {
+ slog.Error("Error creating request:", "error", err)
+ return "", err
+ }
+
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Basic "+authHeader)
+
+ client := http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ slog.Error("Error sending request:", "error", err)
+ return "", err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ slog.Error("Error reading response body:", "error", err)
+ return "", err
+ }
+
+ var tokenResponse TokenResponse
+ err = json.Unmarshal(body, &tokenResponse)
+ if err != nil {
+ slog.Error("Error decoding JSON response:", "error", err)
+ return "", err
+ }
+
+ if tokenResponse.Message != "" {
+ return "", errors.New(tokenResponse.Message)
+ }
+
+ return tokenResponse.SHA1, nil
+}
+
+
+
package gitea
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func (c *Client) GetRepoCommits(ctx context.Context, req gitserver.GetRepoCommitsReq) ([]types.Commit, *types.RepoPageOpts, error) {
+ var commits []types.Commit
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ giteaCommits, response, err := c.giteaClient.ListRepoCommits(
+ namespace,
+ req.Name,
+ gitea.ListCommitOptions{
+ ListOptions: gitea.ListOptions{
+ PageSize: req.Per,
+ Page: req.Page,
+ },
+ SpeedUpOtions: gitea.SpeedUpOtions{
+ DisableStat: true,
+ DisableVerification: true,
+ DisableFiles: true,
+ },
+ SHA: req.Ref,
+ },
+ )
+
+ if err != nil {
+ return nil, nil, err
+ }
+ var commitPageOpt types.RepoPageOpts
+ commitPageOpt.PageCount, err = strconv.Atoi(response.Header.Get(gitserver.Git_Header_X_Pagecount))
+ if err != nil {
+ return nil, nil, err
+ }
+ commitPageOpt.Total, err = strconv.Atoi(response.Header.Get(gitserver.Git_Header_X_Total))
+ if err != nil {
+ return nil, nil, err
+ }
+
+ for _, giteaCommit := range giteaCommits {
+ commits = append(commits, types.Commit{
+ ID: giteaCommit.SHA,
+ CommitterName: giteaCommit.RepoCommit.Committer.Name,
+ CommitterEmail: giteaCommit.RepoCommit.Committer.Email,
+ CommitterDate: giteaCommit.RepoCommit.Committer.Date,
+ CreatedAt: giteaCommit.CommitMeta.Created.String(),
+ Message: giteaCommit.RepoCommit.Message,
+ AuthorName: giteaCommit.RepoCommit.Author.Name,
+ AuthorEmail: giteaCommit.RepoCommit.Author.Email,
+ AuthoredDate: giteaCommit.RepoCommit.Author.Date,
+ })
+ }
+ return commits, &commitPageOpt, nil
+}
+
+func (c *Client) GetRepoLastCommit(ctx context.Context, req gitserver.GetRepoLastCommitReq) (*types.Commit, error) {
+ var commit *types.Commit
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ giteaCommit, _, err := c.giteaClient.GetSingleCommit(
+ namespace,
+ req.Name,
+ req.Ref,
+ gitea.SpeedUpOtions{
+ DisableStat: true,
+ DisableVerification: true,
+ DisableFiles: true,
+ },
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ commit = &types.Commit{
+ ID: giteaCommit.SHA,
+ CommitterName: giteaCommit.RepoCommit.Committer.Name,
+ CommitterEmail: giteaCommit.RepoCommit.Committer.Email,
+ CommitterDate: giteaCommit.RepoCommit.Committer.Date,
+ CreatedAt: giteaCommit.CommitMeta.Created.String(),
+ Message: giteaCommit.RepoCommit.Message,
+ AuthorName: giteaCommit.RepoCommit.Author.Name,
+ AuthorEmail: giteaCommit.RepoCommit.Author.Email,
+ AuthoredDate: giteaCommit.RepoCommit.Author.Date,
+ }
+ return commit, nil
+}
+
+func (c *Client) GetCommitDiff(ctx context.Context, req gitserver.GetRepoLastCommitReq) ([]byte, error) {
+ // namespace is user of gitea
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ data, response, err := c.giteaClient.GetCommitDiff(namespace, req.Name, req.Ref)
+ // response is instance of *gitea.Response
+ if err != nil {
+ slog.Error("Fail to get commit diff", slog.Any("user", namespace), slog.Any("repo", req.Name), slog.Any("commit id", req.Ref), slog.Any("response", response))
+ }
+ return data, err
+}
+
+func (c *Client) GetSingleCommit(ctx context.Context, req gitserver.GetRepoLastCommitReq) (*types.CommitResponse, error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ commit, response, err := c.giteaClient.GetSingleCommit(
+ namespace,
+ req.Name,
+ req.Ref,
+ gitea.SpeedUpOtions{
+ DisableStat: false,
+ DisableVerification: false,
+ DisableFiles: false,
+ },
+ )
+ if err != nil {
+ slog.Error("Fail to get single commit", slog.Any("user", namespace), slog.Any("repo", req.Name), slog.Any("commit id", req.Ref), slog.Any("response", response))
+ }
+
+ diff, err := c.GetCommitDiff(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get git %s repository %s commit id '%s' diff, error: %w", req.RepoType, req.Name, req.Ref, err)
+ }
+
+ commitFiles := []string{}
+ if commit.Files != nil {
+ for _, file := range commit.Files {
+ commitFiles = append(commitFiles, file.Filename)
+ }
+ }
+ commitParents := []*types.CommitMeta{}
+ if commit.Parents != nil {
+ for _, parent := range commit.Parents {
+ commitParents = append(commitParents, &types.CommitMeta{
+ SHA: parent.SHA,
+ })
+ }
+ }
+ commitStats := &types.CommitStats{}
+ if commit.Stats != nil {
+ commitStats.Total = commit.Stats.Total
+ commitStats.Additions = commit.Stats.Additions
+ commitStats.Deletions = commit.Stats.Deletions
+ }
+
+ commitResponse := &types.CommitResponse{
+ Commit: &types.Commit{
+ ID: commit.SHA,
+ AuthorName: commit.RepoCommit.Author.Name,
+ AuthorEmail: commit.RepoCommit.Author.Email,
+ AuthoredDate: commit.RepoCommit.Author.Date,
+ CommitterName: commit.RepoCommit.Committer.Name,
+ CommitterEmail: commit.RepoCommit.Committer.Email,
+ CommitterDate: commit.RepoCommit.Committer.Date,
+ Message: commit.RepoCommit.Message,
+ CreatedAt: commit.CommitMeta.Created.Format("2006-01-02 15:04:05"),
+ },
+ Files: commitFiles,
+ Parents: commitParents,
+ Diff: diff,
+ Stats: commitStats,
+ }
+ return commitResponse, err
+}
+
+func (c *Client) GetDiffBetweenTwoCommits(ctx context.Context, req gitserver.GetDiffBetweenTwoCommitsReq) (*types.GiteaCallbackPushReq, error) {
+ return nil, nil
+}
+
+
+
package gitea
+
+import (
+ "context"
+ "encoding/base64"
+ "errors"
+ "io"
+ "log/slog"
+ "strings"
+ "time"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+const (
+ LFSPrefix = "version https://git-lfs.github.com/spec/v1"
+ NonLFSFileSizeLimit = 10485760
+)
+
+func (c *Client) GetRepoFileTree(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ return c.getRepoDir(namespace, req.Name, req.Ref, req.Path)
+}
+
+func (c *Client) getRepoDir(namespace, name, ref, path string) (files []*types.File, err error) {
+ giteaEntries, _, err := c.giteaClient.GetDir(namespace, name, ref, path)
+ if err != nil {
+ return
+ }
+ for _, entry := range giteaEntries {
+ f := &types.File{
+ Name: entry.Name,
+ Path: strings.TrimPrefix(entry.Path, "/"),
+ Type: entry.Type,
+ Lfs: entry.IsLfs,
+ LfsRelativePath: entry.LfsRelativePath,
+ Size: entry.Size,
+ Commit: types.Commit{
+ Message: entry.CommitMsg, ID: entry.SHA, CommitterDate: entry.CommitterDate.String(),
+ },
+ Mode: entry.Mode,
+ SHA: entry.SHA,
+ URL: entry.URL,
+ LastCommitSHA: entry.LastCommitSHA,
+ }
+ if entry.Type == "tree" {
+ f.Type = "dir"
+ } else {
+ f.Type = "file"
+ }
+
+ files = append(files, f)
+
+ }
+
+ return files, nil
+}
+
+func (c *Client) GetRepoFileRaw(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (string, error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ giteaFileData, _, err := c.giteaClient.GetFile(namespace, req.Name, req.Ref, req.Path)
+ if err != nil {
+ return "", err
+ }
+ return string(giteaFileData), nil
+}
+
+func (c *Client) GetRepoFileReader(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (io.ReadCloser, int64, error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ entries, _, err := c.giteaClient.GetDir(namespace, req.Name, req.Ref, req.Path)
+ if err != nil {
+ return nil, 0, err
+ }
+ if entries[0].Size > NonLFSFileSizeLimit {
+ return nil, 0, errors.New("file is larger than 10MB")
+ }
+ giteaFileReader, _, err := c.giteaClient.GetFileReader(namespace, req.Name, req.Ref, req.Path)
+ if err != nil {
+ return nil, 0, err
+ }
+ return giteaFileReader, entries[0].Size, nil
+}
+
+func (c *Client) GetRepoLfsFileRaw(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (io.ReadCloser, error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ r, _, err := c.giteaClient.GetFileReader(namespace, req.Name, req.Ref, req.Path, true)
+ return r, err
+}
+
+func (c *Client) GetRepoFileContents(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (*types.File, error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ file, err := c.getFileContents(namespace, req.Name, req.Ref, req.Path)
+ if err != nil {
+ if err.Error() == "GetContentsOrList" {
+ // not found message of gitea
+ return nil, err
+ }
+ return nil, errors.New("failed to get repo file contents")
+ }
+ commit, _, err := c.giteaClient.GetSingleCommit(namespace, req.Name, file.LastCommitSHA, gitea.SpeedUpOtions{
+ DisableStat: true,
+ DisableVerification: true,
+ DisableFiles: true,
+ })
+ if err != nil {
+ return nil, errors.New("failed to get repo file last commit")
+ }
+
+ file.Commit = types.Commit{
+ ID: commit.SHA,
+ CommitterName: commit.RepoCommit.Committer.Name,
+ CommitterEmail: commit.RepoCommit.Committer.Email,
+ CommitterDate: commit.RepoCommit.Committer.Date,
+ CreatedAt: commit.Created.Format("2024-02-26T15:05:35+08:00"),
+ Message: commit.RepoCommit.Message,
+ AuthorName: commit.RepoCommit.Author.Name,
+ AuthorEmail: commit.RepoCommit.Author.Email,
+ AuthoredDate: commit.RepoCommit.Author.Date,
+ }
+ return file, nil
+}
+
+func (c *Client) CreateRepoFile(req *types.CreateFileReq) (err error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ _, _, err = c.giteaClient.CreateFile(namespace, req.Name, req.FilePath, gitea.CreateFileOptions{
+ FileOptions: gitea.FileOptions{
+ Message: req.Message,
+ BranchName: req.Branch,
+ NewBranchName: req.NewBranch,
+ Author: gitea.Identity{
+ Name: req.Username,
+ Email: req.Email,
+ },
+ Committer: gitea.Identity{
+ Name: req.Username,
+ Email: req.Email,
+ },
+ Dates: gitea.CommitDateOptions{
+ Author: time.Now(),
+ Committer: time.Now(),
+ },
+ },
+ Content: req.Content,
+ })
+ return
+}
+
+func (c *Client) UpdateRepoFile(req *types.UpdateFileReq) (err error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ _, _, err = c.giteaClient.UpdateFile(namespace, req.Name, req.FilePath, gitea.UpdateFileOptions{
+ FileOptions: gitea.FileOptions{
+ Message: req.Message,
+ BranchName: req.Branch,
+ NewBranchName: req.NewBranch,
+ Author: gitea.Identity{
+ Name: req.Username,
+ Email: req.Email,
+ },
+ Committer: gitea.Identity{
+ Name: req.Username,
+ Email: req.Email,
+ },
+ Dates: gitea.CommitDateOptions{
+ Author: time.Now(),
+ Committer: time.Now(),
+ },
+ },
+ SHA: req.SHA,
+ Content: req.Content,
+ FromPath: req.OriginPath,
+ })
+ return
+}
+
+func (c *Client) DeleteRepoFile(req *types.DeleteFileReq) (err error) {
+ return nil
+}
+
+func (c *Client) getFileContents(owner, repo, ref, path string) (*types.File, error) {
+ var content string
+ /* Example file content from gitea
+ {
+ "name": "model-00001-of-00002.safetensors",
+ "path": "model-00001-of-00002.safetensors",
+ "sha": "5824b5e840050f193d3d9091a6b5dbb3c33fb0f3",
+ "last_commit_sha": "d682aedec5182964a9c17e9dd9d5f437847a8c43",
+ "type": "file",
+ "size": 135,
+ "encoding": "base64",
+ "content": "dmVyc2lvbiBodHRwczovL2dpdC1sZnMuZ2l0aHViLmNvbS9zcGVjL3YxCm9pZCBzaGEyNTY6ODYwMmQ0ZDNiMTRmNTg2NTIwZDFmNzY1MDkxZDlmM2E0ZmViMWM1Nzg2NDQ4YzYwMGQwMThkYjcyMTZmNzIzNQpzaXplIDQ5ODI0Njc4NjQK",
+ "target": null,
+ "url": "http://portal-stg.opencsg.com:3001/api/v1/repos/models_wayne/phi-2/contents/model-00001-of-00002.safetensors?ref=main",
+ "html_url": "http://portal-stg.opencsg.com:3001/models_wayne/phi-2/src/branch/main/model-00001-of-00002.safetensors",
+ "git_url": "http://portal-stg.opencsg.com:3001/api/v1/repos/models_wayne/phi-2/git/blobs/5824b5e840050f193d3d9091a6b5dbb3c33fb0f3",
+ "download_url": "http://portal-stg.opencsg.com:3001/models_wayne/phi-2/raw/branch/main/model-00001-of-00002.safetensors",
+ "submodule_git_url": null,
+ "_links": {
+ "self": "http://portal-stg.opencsg.com:3001/api/v1/repos/models_wayne/phi-2/contents/model-00001-of-00002.safetensors?ref=main",
+ "git": "http://portal-stg.opencsg.com:3001/api/v1/repos/models_wayne/phi-2/git/blobs/5824b5e840050f193d3d9091a6b5dbb3c33fb0f3",
+ "html": "http://portal-stg.opencsg.com:3001/models_wayne/phi-2/src/branch/main/model-00001-of-00002.safetensors"
+ }
+ }
+ */
+ fileContent, _, err := c.giteaClient.GetContents(owner, repo, ref, path)
+ if err != nil {
+ slog.Error("Failed to get contents from gitea", slog.Any("error", err), slog.String("owner", owner), slog.String("repo", repo), slog.String("ref", ref), slog.String("path", path))
+ return nil, err
+ }
+ if fileContent.Content != nil {
+ content = *fileContent.Content
+ }
+
+ f := &types.File{
+ Name: fileContent.Name,
+ Type: fileContent.Type,
+ Size: fileContent.Size,
+ SHA: fileContent.SHA,
+ Path: fileContent.Path,
+ Content: content,
+ LastCommitSHA: fileContent.LastCommitSHA,
+ }
+
+ // base64 decode
+ contentDecoded, _ := base64.StdEncoding.DecodeString(f.Content)
+ lfsPointer, err := ReadPointerFromBuffer(contentDecoded)
+ // not a lfs pointer, return file content directly
+ if err != nil || !lfsPointer.IsValid() {
+ slog.Info("Failed to parse lsf pointer", slog.Any("error", err), slog.Bool("isValidPointer", lfsPointer.IsValid()))
+ return f, nil
+ }
+
+ f.Lfs = true
+ f.LfsRelativePath = lfsPointer.RelativePath()
+ // file content is the lfs pointer if the file is a lfs file
+ f.LfsPointerSize = int(fileContent.Size)
+ f.Size = lfsPointer.Size
+ return f, nil
+}
+
+func (c *Client) GetRepoAllFiles(ctx context.Context, req gitserver.GetRepoAllFilesReq) ([]*types.File, error) {
+ return nil, nil
+}
+
+func (c *Client) GetRepoAllLfsPointers(ctx context.Context, req gitserver.GetRepoAllFilesReq) ([]*types.LFSPointer, error) {
+ return nil, nil
+}
+
+
+
package gitea
+
+import (
+ "context"
+ "io"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+)
+
+func (c *Client) InfoRefsResponse(ctx context.Context, req gitserver.InfoRefsReq) (io.Reader, error) {
+ return nil, nil
+}
+
+func (c *Client) UploadPack(ctx context.Context, req gitserver.UploadPackReq) error {
+ return nil
+}
+
+func (c *Client) ReceivePack(ctx context.Context, req gitserver.ReceivePackReq) error {
+ return nil
+}
+
+
+
/*
+Content of this file is totally copied from gitea project
+*/
+
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package gitea
+
+import (
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/minio/sha256-simd"
+)
+
+// Pointer contains LFS pointer data
+type Pointer struct {
+ Oid string `json:"oid" xorm:"UNIQUE(s) INDEX NOT NULL"`
+ Size int64 `json:"size" xorm:"NOT NULL"`
+}
+
+const (
+ blobSizeCutoff = 1024
+
+ // MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
+ // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
+ MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
+
+ // MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
+ MetaFileOidPrefix = "oid sha256:"
+)
+
+var (
+ // ErrMissingPrefix occurs if the content lacks the LFS prefix
+ ErrMissingPrefix = errors.New("content lacks the LFS prefix")
+
+ // ErrInvalidStructure occurs if the content has an invalid structure
+ ErrInvalidStructure = errors.New("content has an invalid structure")
+
+ // ErrInvalidOIDFormat occurs if the oid has an invalid format
+ ErrInvalidOIDFormat = errors.New("OID has an invalid format")
+)
+
+// ReadPointer tries to read LFS pointer data from the reader
+func ReadPointer(reader io.Reader) (Pointer, error) {
+ buf := make([]byte, blobSizeCutoff)
+ n, err := io.ReadFull(reader, buf)
+ if err != nil && err != io.ErrUnexpectedEOF {
+ return Pointer{}, err
+ }
+ buf = buf[:n]
+
+ return ReadPointerFromBuffer(buf)
+}
+
+var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
+
+// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
+func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
+ var p Pointer
+
+ headString := string(buf)
+ if !strings.HasPrefix(headString, MetaFileIdentifier) {
+ return p, ErrMissingPrefix
+ }
+
+ splitLines := strings.Split(headString, "\n")
+ if len(splitLines) < 3 {
+ return p, ErrInvalidStructure
+ }
+
+ oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
+ if len(oid) != 64 || !oidPattern.MatchString(oid) {
+ return p, ErrInvalidOIDFormat
+ }
+ size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+ if err != nil {
+ return p, err
+ }
+
+ p.Oid = oid
+ p.Size = size
+
+ return p, nil
+}
+
+// IsValid checks if the pointer has a valid structure.
+// It doesn't check if the pointed-to-content exists.
+func (p Pointer) IsValid() bool {
+ if len(p.Oid) != 64 {
+ return false
+ }
+ if !oidPattern.MatchString(p.Oid) {
+ return false
+ }
+ if p.Size < 0 {
+ return false
+ }
+ return true
+}
+
+// StringContent returns the string representation of the pointer
+// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer
+func (p Pointer) StringContent() string {
+ return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size)
+}
+
+// RelativePath returns the relative storage path of the pointer
+func (p Pointer) RelativePath() string {
+ if len(p.Oid) < 5 {
+ return p.Oid
+ }
+
+ return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
+}
+
+func (p Pointer) LogString() string {
+ if p.Oid == "" && p.Size == 0 {
+ return "<LFSPointer empty>"
+ }
+ return fmt.Sprintf("<LFSPointer %s:%d>", p.Oid, p.Size)
+}
+
+// GeneratePointer generates a pointer for arbitrary content
+func GeneratePointer(content io.Reader) (Pointer, error) {
+ h := sha256.New()
+ c, err := io.Copy(h, content)
+ if err != nil {
+ return Pointer{}, err
+ }
+ sum := h.Sum(nil)
+ return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil
+}
+
+
+
package gitea
+
+import (
+ "context"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func (c *Client) CreateMirrorRepo(ctx context.Context, req gitserver.CreateMirrorRepoReq) (int64, error) {
+ task, _, err := c.giteaClient.MigrateRepo(gitea.MigrateRepoOption{
+ RepoName: req.Name,
+ RepoOwner: common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType)),
+ CloneAddr: req.CloneUrl,
+ Service: gitea.GitServicePlain,
+ AuthUsername: req.Username,
+ AuthPassword: req.AccessToken,
+ Mirror: true,
+ Private: req.Private,
+ Description: req.Description,
+ Wiki: false,
+ Milestones: false,
+ Labels: false,
+ Issues: false,
+ PullRequests: false,
+ Releases: false,
+ LFS: true,
+ MirrorInterval: "0",
+ Token: req.MirrorToken,
+ })
+ if err != nil {
+ return 0, err
+ }
+ return task.ID, nil
+}
+
+func (c *Client) GetMirrorTaskInfo(ctx context.Context, taskId int64) (*gitserver.MirrorTaskInfo, error) {
+ ts, _, err := c.giteaClient.GetUserTaskInfo(taskId)
+ if err != nil {
+ return nil, err
+ }
+
+ mti := &gitserver.MirrorTaskInfo{
+ Status: gitserver.TaskStatus(ts.Status),
+ Message: ts.Message,
+ RepoID: ts.RepoID,
+ RepoName: ts.RepoName,
+ StartedAt: ts.StartedAt,
+ EndedAt: ts.EndedAt,
+ }
+ return mti, nil
+}
+
+func (c *Client) MirrorSync(ctx context.Context, req gitserver.MirrorSyncReq) error {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ _, err := c.giteaClient.MirrorSync(namespace, req.Name)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+
+
package gitea
+
+import (
+ "errors"
+ "log/slog"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+// FixOrganization recreate organization data, ignore data duplication error
+// !should only be used for online data fixing
+func (c *Client) FixOrganization(req *types.CreateOrgReq, user database.User) error {
+ var errs error
+ orgNames := c.getTargetOrgs(req.Name)
+ for _, orgName := range orgNames {
+ _, _, err := c.giteaClient.AdminCreateOrg(
+ user.Username,
+ gitea.CreateOrgOption{
+ Name: orgName,
+ Description: req.Description,
+ FullName: req.Nickname,
+ },
+ )
+ if err != nil {
+ errs = errors.Join(err)
+ slog.Error("fix gitea organization failed", slog.String("orgName", orgName), slog.String("user", user.Username),
+ slog.String("error", err.Error()))
+ } else {
+ slog.Info("fix gitea organization success", slog.String("orgName", req.Name), slog.String("user", user.Username))
+ }
+ }
+
+ return errs
+}
+
+func (c *Client) CreateOrganization(req *types.CreateOrgReq, user database.User) (org *database.Organization, err error) {
+ orgNames := c.getTargetOrgs(req.Name)
+ for _, orgName := range orgNames {
+ _, _, err := c.giteaClient.AdminCreateOrg(
+ user.Username,
+ gitea.CreateOrgOption{
+ Name: orgName,
+ Description: req.Description,
+ FullName: req.Nickname,
+ },
+ )
+ if err != nil {
+ slog.Error("create gitea organization failed", slog.String("orgName", orgName), slog.String("userName", user.Username))
+ return nil, err
+ }
+ }
+
+ org = &database.Organization{
+ Name: req.Name,
+ Nickname: req.Nickname,
+ Description: req.Description,
+ User: &user,
+ UserID: user.ID,
+ }
+
+ return
+}
+
+func (c *Client) DeleteOrganization(name string) (err error) {
+ orgNames := c.getTargetOrgs(name)
+ for _, orgName := range orgNames {
+ _, err = c.giteaClient.DeleteOrg(orgName)
+ if err != nil {
+ return
+ }
+ }
+
+ return
+}
+
+// TODO:remove param `originOrg`
+func (c *Client) UpdateOrganization(req *types.EditOrgReq, originOrg *database.Organization) (org *database.Organization, err error) {
+ orgNames := c.getTargetOrgs(req.Name)
+
+ for _, orgName := range orgNames {
+ _, err = c.giteaClient.EditOrg(
+ orgName,
+ gitea.EditOrgOption{
+ FullName: *req.Nickname,
+ Description: *req.Description,
+ },
+ )
+ if err != nil {
+ return
+ }
+ }
+
+ return originOrg, nil
+}
+
+func (c *Client) getTargetOrgs(org string) []string {
+ orgs := [4]string{
+ common.WithPrefix(org, DatasetOrgPrefix),
+ common.WithPrefix(org, ModelOrgPrefix),
+ common.WithPrefix(org, SpaceOrgPrefix),
+ common.WithPrefix(org, CodeOrgPrefix),
+ }
+ return orgs[:]
+}
+
+
+
package gitea
+
+import (
+ "context"
+ "log/slog"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+const (
+ ModelOrgPrefix = "models_"
+ DatasetOrgPrefix = "datasets_"
+ SpaceOrgPrefix = "spaces_"
+ CodeOrgPrefix = "codes_"
+)
+
+func (c *Client) CreateRepo(ctx context.Context, req gitserver.CreateRepoReq) (*gitserver.CreateRepoResp, error) {
+ giteaRepo, _, err := c.giteaClient.CreateOrgRepo(
+ common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType)),
+ gitea.CreateRepoOption{
+ Name: req.Name,
+ Description: req.Description,
+ Private: req.Private,
+ IssueLabels: req.Labels,
+ License: req.License,
+ Readme: req.Readme,
+ DefaultBranch: req.DefaultBranch,
+ },
+ )
+ if err != nil {
+ slog.Error("fail to call gitea to create repository", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, err
+ }
+
+ resp := &gitserver.CreateRepoResp{
+ Username: req.Username,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Nickname: req.Nickname,
+ Description: req.Description,
+ Labels: req.Labels,
+ License: req.License,
+ DefaultBranch: giteaRepo.DefaultBranch,
+ RepoType: req.RepoType,
+ GitPath: giteaRepo.FullName,
+ SshCloneURL: giteaRepo.SSHURL,
+ HttpCloneURL: common.PortalCloneUrl(giteaRepo.CloneURL, req.RepoType, c.config.GitServer.URL, c.config.Frontend.URL),
+ Private: req.Private,
+ }
+
+ return resp, nil
+}
+
+func (c *Client) UpdateRepo(ctx context.Context, req gitserver.UpdateRepoReq) (*gitserver.CreateRepoResp, error) {
+ giteaRepo, _, err := c.giteaClient.EditRepo(
+ common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType)),
+ req.Name,
+ gitea.EditRepoOption{
+ Description: gitea.OptionalString(req.Description),
+ Private: gitea.OptionalBool(req.Private),
+ DefaultBranch: gitea.OptionalString(req.DefaultBranch),
+ },
+ )
+ if err != nil {
+ slog.Error("fail to call gitea to update repository", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, err
+ }
+
+ resp := &gitserver.CreateRepoResp{
+ Nickname: giteaRepo.FullName,
+ Description: giteaRepo.Description,
+ DefaultBranch: giteaRepo.DefaultBranch,
+ RepoType: req.RepoType,
+ GitPath: giteaRepo.FullName,
+ Private: giteaRepo.Private,
+ }
+ return resp, nil
+}
+
+func (c *Client) DeleteRepo(ctx context.Context, req gitserver.DeleteRepoReq) error {
+ giteaNamespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ _, err := c.giteaClient.DeleteRepo(giteaNamespace, req.Name)
+ return err
+}
+
+func (c *Client) GetRepo(ctx context.Context, req gitserver.GetRepoReq) (*gitserver.CreateRepoResp, error) {
+ giteaRepo, _, err := c.giteaClient.GetRepo(
+ common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType)),
+ req.Name,
+ )
+ if err != nil {
+ slog.Error("fail to call get gitea repository", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, err
+ }
+
+ resp := &gitserver.CreateRepoResp{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Nickname: giteaRepo.FullName,
+ Description: giteaRepo.Description,
+ DefaultBranch: giteaRepo.DefaultBranch,
+ RepoType: req.RepoType,
+ GitPath: giteaRepo.FullName,
+ SshCloneURL: giteaRepo.SSHURL,
+ HttpCloneURL: common.PortalCloneUrl(giteaRepo.CloneURL, req.RepoType, c.config.GitServer.URL, c.config.Frontend.URL),
+ Private: giteaRepo.Private,
+ }
+
+ return resp, nil
+}
+
+
+
package gitea
+
+import (
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func (c *Client) CreateSSHKey(req *types.CreateSSHKeyRequest) (token *database.SSHKey, err error) {
+ giteaSSHKey, _, err := c.giteaClient.AdminCreateUserPublicKey(
+ req.Username,
+ gitea.CreateKeyOption{
+ Title: req.Name,
+ Key: req.Content,
+ },
+ )
+
+ if err != nil {
+ return
+ }
+
+ token = &database.SSHKey{
+ GitID: giteaSSHKey.ID,
+ Name: req.Name,
+ Content: giteaSSHKey.Key,
+ }
+
+ return
+}
+
+//List all SSH keys from gitea
+
+// func (c *Client) ListSSHKeys(username string, per, page int) (tokens []*database.SSHKey, err error) {
+// giteaSSHKeys, _, err := c.giteaClient.ListPublicKeys(
+// username,
+// gitea.ListPublicKeysOptions{
+// ListOptions: gitea.ListOptions{
+// Page: page,
+// PageSize: per,
+// },
+// },
+// )
+
+// if err != nil {
+// return
+// }
+
+// for _, giteaSSHKey := range giteaSSHKeys {
+// tokens = append(tokens, &database.SSHKey{
+// GID: int(giteaSSHKey.ID),
+// Name: giteaSSHKey.Title,
+// Content: giteaSSHKey.Key,
+// })
+// }
+
+// return
+// }
+
+func (c *Client) DeleteSSHKey(id int) (err error) {
+ _, err = c.giteaClient.DeletePublicKey(int64(id))
+ return
+}
+
+
+
package gitea
+
+import (
+ "context"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func (c *Client) GetRepoTags(ctx context.Context, req gitserver.GetRepoTagsReq) (tags []*types.Tag, err error) {
+ namespace := common.WithPrefix(req.Namespace, repoPrefixByType(req.RepoType))
+ giteaTags, _, err := c.giteaClient.ListRepoTags(
+ namespace,
+ req.Name,
+ gitea.ListRepoTagsOptions{
+ ListOptions: gitea.ListOptions{
+ PageSize: req.Per,
+ Page: req.Page,
+ },
+ },
+ )
+ if err != nil {
+ return
+ }
+ for _, giteaTag := range giteaTags {
+ tag := &types.Tag{
+ Name: giteaTag.Name,
+ Message: giteaTag.Message,
+ Commit: types.DatasetTagCommit{
+ ID: giteaTag.Commit.SHA,
+ },
+ }
+ tags = append(tags, tag)
+ }
+ return
+}
+
+
+
package gitea
+
+import (
+ "opencsg.com/csghub-server/common/types"
+)
+
+func repoPrefixByType(repoType types.RepositoryType) string {
+ var prefix string
+ switch repoType {
+ case types.ModelRepo:
+ prefix = ModelOrgPrefix
+ case types.DatasetRepo:
+ prefix = DatasetOrgPrefix
+ case types.SpaceRepo:
+ prefix = SpaceOrgPrefix
+ case types.CodeRepo:
+ prefix = CodeOrgPrefix
+ }
+
+ return prefix
+}
+
+
+
package gitea
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/sha1"
+ "encoding/hex"
+ "math/big"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+func (c *Client) CreateUser(u gitserver.CreateUserRequest) (user *gitserver.CreateUserResponse, err error) {
+ password, err := generateRandomPassword(12)
+ if err != nil {
+ return
+ }
+
+ giteaUser, _, err := c.giteaClient.AdminCreateUser(
+ gitea.CreateUserOption{
+ FullName: u.Nickname,
+ Username: u.Username,
+ Email: u.Email,
+ Password: password,
+ MustChangePassword: gitea.OptionalBool(false),
+ SendNotify: false,
+ },
+ )
+
+ if err != nil {
+ return
+ }
+
+ err = c.createOrgsForUser(giteaUser)
+
+ if err != nil {
+ return
+ }
+
+ password = calculateSHA1(password)
+ user = &gitserver.CreateUserResponse{
+ Email: giteaUser.Email,
+ GitID: giteaUser.ID,
+ Username: giteaUser.UserName,
+ NickName: giteaUser.FullName,
+ Password: password,
+ }
+
+ return
+}
+
+func (c *Client) UpdateUser(u *types.UpdateUserRequest, user *database.User) (*database.User, error) {
+ _, err := c.giteaClient.AdminEditUser(
+ u.Username,
+ gitea.EditUserOption{
+ LoginName: u.Username,
+ FullName: u.Nickname,
+ Email: u.Email,
+ },
+ )
+
+ user.NickName = *u.Nickname
+ user.Email = *u.Email
+ return user, err
+}
+
+func (c *Client) UpdateUserV2(u gitserver.UpdateUserRequest) error {
+ //nothing to update
+ if u.Nickname == nil && u.Email == nil {
+ return nil
+ }
+
+ opt := gitea.EditUserOption{
+ LoginName: u.Username,
+ }
+ if u.Nickname != nil {
+ opt.FullName = u.Nickname
+ }
+
+ if u.Email != nil {
+ opt.Email = u.Email
+ }
+ _, err := c.giteaClient.AdminEditUser(
+ u.Username,
+ opt,
+ )
+
+ return err
+}
+
+// Random password generator
+func generateRandomPassword(length int) (string, error) {
+ charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()_+"
+ charsetLength := big.NewInt(int64(len(charset)))
+ password := make([]byte, length)
+
+ for i := range password {
+ randomIndex, err := rand.Int(rand.Reader, charsetLength)
+ if err != nil {
+ return "", err
+ }
+ password[i] = charset[randomIndex.Int64()]
+ }
+
+ return string(password), nil
+}
+
+// Create three orgs for user
+func (c *Client) createOrgsForUser(user *gitea.User) (err error) {
+ orgNames := []string{
+ common.WithPrefix(user.UserName, ModelOrgPrefix),
+ common.WithPrefix(user.UserName, DatasetOrgPrefix),
+ common.WithPrefix(user.UserName, SpaceOrgPrefix),
+ common.WithPrefix(user.UserName, CodeOrgPrefix),
+ }
+
+ for _, orgName := range orgNames {
+ _, _, err = c.giteaClient.AdminCreateOrg(
+ user.UserName,
+ gitea.CreateOrgOption{
+ Name: orgName,
+ FullName: orgName,
+ },
+ )
+ if err != nil {
+ return
+ }
+ }
+
+ return
+}
+
+// Create gitea orgs for user to store different type repositories
+func (c *Client) FixUserData(ctx context.Context, userName string) (err error) {
+ orgNames := []string{
+ common.WithPrefix(userName, ModelOrgPrefix),
+ common.WithPrefix(userName, DatasetOrgPrefix),
+ common.WithPrefix(userName, SpaceOrgPrefix),
+ common.WithPrefix(userName, CodeOrgPrefix),
+ }
+
+ for _, orgName := range orgNames {
+ _, _, err = c.giteaClient.AdminCreateOrg(
+ userName,
+ gitea.CreateOrgOption{
+ Name: orgName,
+ FullName: orgName,
+ },
+ )
+ }
+
+ return
+}
+
+func calculateSHA1(input string) string {
+ hasher := sha1.New()
+ hasher.Write([]byte(input))
+ hashInBytes := hasher.Sum(nil)
+ hashString := hex.EncodeToString(hashInBytes)
+
+ return hashString
+}
+
+
+
package gitea
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+)
+
+const (
+ ModelOrgPrefix = "models_"
+ DatasetOrgPrefix = "datasets_"
+ SpaceOrgPrefix = "spaces_"
+ CodeOrgPrefix = "codes_"
+)
+
+type Client struct {
+ giteaClient *gitea.Client
+ config *config.Config
+}
+
+type Response struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+ Data any `json:"data"`
+}
+
+type TokenResponse struct {
+ SHA1 string `json:"sha1"`
+ Message string `json:"message"`
+}
+
+func NewClient(config *config.Config) (client *Client, err error) {
+ ctx := context.Background()
+ token, err := findOrCreateAccessToken(ctx, config)
+ if err != nil {
+ slog.Error("Failed to find or create token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+ giteaClient, err := gitea.NewClient(
+ config.GitServer.Host,
+ gitea.SetContext(ctx),
+ gitea.SetToken(token.Token),
+ gitea.SetBasicAuth(config.GitServer.Username, config.GitServer.Password),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return &Client{giteaClient: giteaClient, config: config}, nil
+}
+
+func findOrCreateAccessToken(ctx context.Context, config *config.Config) (*database.GitServerAccessToken, error) {
+ gs := database.NewGitServerAccessTokenStore()
+ tokens, err := gs.FindByType(ctx, "git")
+ if err != nil {
+ slog.Error("Fail to get git server access token from database", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+
+ if len(tokens) == 0 {
+ access_token, err := generateAccessTokenFromGitea(config)
+ if err != nil {
+ slog.Error("Fail to create git server access token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+ gToken := &database.GitServerAccessToken{
+ Token: access_token,
+ }
+
+ gToken, err = gs.Create(ctx, gToken)
+ if err != nil {
+ slog.Error("Fail to create git server access token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+
+ return gToken, nil
+ }
+ return &tokens[0], nil
+}
+
+func encodeCredentials(username, password string) string {
+ credentials := username + ":" + password
+ return base64.StdEncoding.EncodeToString([]byte(credentials))
+}
+
+func generateAccessTokenFromGitea(config *config.Config) (string, error) {
+ username := config.GitServer.Username
+ password := config.GitServer.Password
+ giteaUrl := fmt.Sprintf("%s/api/v1/users/%s/tokens", config.GitServer.Host, username)
+ authHeader := encodeCredentials(username, password)
+ data := map[string]any{
+ "name": "access_token",
+ "scopes": []string{
+ "write:user",
+ "write:admin",
+ "write:organization",
+ "write:repository",
+ },
+ }
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ slog.Error("Error encoding JSON data:", "error", err)
+ return "", err
+ }
+
+ req, err := http.NewRequest("POST", giteaUrl, bytes.NewBuffer(jsonData))
+ if err != nil {
+ slog.Error("Error creating request:", "error", err)
+ return "", err
+ }
+
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Basic "+authHeader)
+
+ client := http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ slog.Error("Error sending request:", "error", err)
+ return "", err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ slog.Error("Error reading response body:", "error", err)
+ return "", err
+ }
+
+ var tokenResponse TokenResponse
+ err = json.Unmarshal(body, &tokenResponse)
+ if err != nil {
+ slog.Error("Error decoding JSON response:", "error", err)
+ return "", err
+ }
+
+ if tokenResponse.Message != "" {
+ return "", errors.New(tokenResponse.Message)
+ }
+
+ return tokenResponse.SHA1, nil
+}
+
+
+
package gitea
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+var _ membership.GitMemerShip = (*Client)(nil)
+
+func (c *Client) AddRoles(ctx context.Context, org string, roles []membership.Role) error {
+ var errs error
+ for _, to := range c.getTargetOrgs(org) {
+ errs = errors.Join(errs,
+ c.addRoles(ctx, to, roles))
+ }
+ return errs
+}
+
+func (c *Client) addRoles(ctx context.Context, org string, roles []membership.Role) error {
+ var errs error
+ for _, role := range roles {
+ opt := c.getTeamOptByRole(role)
+ _, resp, err := c.giteaClient.CreateTeam(org, opt)
+ if err != nil {
+ slog.Error("gitea create team failed", slog.String("org", org), slog.Any("body", resp.Body),
+ slog.Int("code", resp.StatusCode), slog.String("error", err.Error()))
+ errs = errors.Join(errs, err)
+ }
+ }
+ return errs
+}
+
+func (c *Client) getTeamOptByRole(role membership.Role) gitea.CreateTeamOption {
+ var opt gitea.CreateTeamOption
+ opt.Name = string(role)
+ opt.IncludesAllRepositories = true
+ opt.Units = append(opt.Units, gitea.RepoUnitCode)
+ switch role {
+ case membership.RoleAdmin:
+ opt.CanCreateOrgRepo = true
+ opt.Permission = gitea.AccessModeAdmin
+ case membership.RoleWrite:
+ opt.CanCreateOrgRepo = false
+ opt.Permission = gitea.AccessModeWrite
+ case membership.RoleRead:
+ opt.CanCreateOrgRepo = false
+ opt.Permission = gitea.AccessModeRead
+ }
+ return opt
+}
+
+func (c *Client) AddMember(ctx context.Context, org, member string, role membership.Role) error {
+ var err error
+ for _, to := range c.getTargetOrgs(org) {
+ err = errors.Join(err,
+ c.addMember(ctx, to, member, role))
+ }
+
+ return err
+}
+
+func (c *Client) addMember(ctx context.Context, org, member string, role membership.Role) error {
+ // teams are created automatically when create repo
+ t, err := c.findTeam(org, role)
+ if err != nil {
+ slog.ErrorContext(ctx, "fail to get team from gitea", slog.Any("err", err))
+ return err
+ }
+ _, resp, err := c.giteaClient.GetTeamMember(t.ID, member)
+ // silently success if user is already a member
+ if resp.StatusCode == http.StatusOK {
+ return nil
+ }
+ // if not a member
+ if resp.StatusCode == http.StatusNotFound {
+ _, err = c.giteaClient.AddTeamMember(t.ID, member)
+ if err != nil {
+ slog.ErrorContext(ctx, "fail to add team member to gitea", slog.Any("err", err))
+ }
+ return err
+ }
+ // unknown server error happened
+ slog.ErrorContext(ctx, "fail to get team member from gitea", slog.Any("err", err))
+ return err
+}
+
+func (c *Client) RemoveMember(ctx context.Context, org, member string, role membership.Role) error {
+ var err error
+ for _, to := range c.getTargetOrgs(org) {
+ err = errors.Join(err,
+ c.removeMember(ctx, to, member, role))
+ }
+ return err
+}
+
+func (c *Client) removeMember(ctx context.Context, org, member string, role membership.Role) error {
+ t, err := c.findTeam(org, role)
+ if err != nil {
+ return err
+ }
+ u, _, err := c.giteaClient.GetTeamMember(t.ID, member)
+ if err != nil {
+ return err
+ }
+
+ // silently success if user is not a member
+ if u == nil {
+ return nil
+ }
+ _, err = c.giteaClient.RemoveTeamMember(t.ID, member)
+ return err
+}
+
+func (c *Client) IsRole(ctx context.Context, org, member string, role membership.Role) (bool, error) {
+ return false, nil
+}
+
+func (c *Client) getTargetOrgs(org string) []string {
+ orgs := [4]string{
+ common.WithPrefix(org, DatasetOrgPrefix),
+ common.WithPrefix(org, ModelOrgPrefix),
+ common.WithPrefix(org, SpaceOrgPrefix),
+ common.WithPrefix(org, CodeOrgPrefix),
+ }
+ return orgs[:]
+}
+
+func (c *Client) findTeam(org string, role membership.Role) (*gitea.Team, error) {
+ opt := &gitea.SearchTeamsOptions{
+ Query: roleToTeamName(role),
+ ListOptions: gitea.ListOptions{
+ PageSize: 1,
+ },
+ IncludeDescription: false,
+ }
+ teams, _, err := c.giteaClient.SearchOrgTeams(org, opt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to search org team, error:%w", err)
+ }
+ if len(teams) == 0 {
+ return nil, fmt.Errorf("gitea team not found by role:%s", role)
+ }
+
+ t := teams[0]
+ return t, nil
+}
+
+func roleToTeamName(role membership.Role) string {
+ return string(role)
+}
+
+
+
package membership
+
+type Role string
+
+const (
+ RoleUnknown Role = ""
+ RoleRead Role = "read"
+ RoleWrite Role = "write"
+ RoleAdmin Role = "admin"
+)
+
+func (r Role) CanRead() bool {
+ return r != RoleUnknown
+}
+
+func (r Role) CanWrite() bool {
+ return r == RoleWrite || r == RoleAdmin
+}
+
+func (r Role) CanAdmin() bool {
+ return r == RoleAdmin
+}
+
+
+
package git
+
+import (
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/git/membership/gitea"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func NewMemberShip(config config.Config) (membership.GitMemerShip, error) {
+ c, err := gitea.NewClient(&config)
+ return c, err
+}
+
+
+
package git
+
+import (
+ "errors"
+
+ "opencsg.com/csghub-server/builder/git/mirrorserver"
+ "opencsg.com/csghub-server/builder/git/mirrorserver/gitea"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func NewMirrorServer(config *config.Config) (mirrorserver.MirrorServer, error) {
+ if !config.MirrorServer.Enable {
+ return nil, nil
+ }
+ if config.MirrorServer.Type == types.GitServerTypeGitea {
+ mirrorServer, err := gitea.NewMirrorClient(config)
+ return mirrorServer, err
+ }
+
+ //TODO: implement gitaly based mirrorserver
+
+ return nil, errors.New("undefined mirror server type")
+}
+
+
+
package gitea
+
+import (
+ "context"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/git/mirrorserver"
+)
+
+const (
+ ModelOrgPrefix = "models_"
+ DatasetOrgPrefix = "datasets_"
+ SpaceOrgPrefix = "spaces_"
+ CodeOrgPrefix = "codes_"
+ MirrorServerNamespace = "root"
+)
+
+func (c *MirrorClient) CreateMirrorRepo(ctx context.Context, req mirrorserver.CreateMirrorRepoReq) (int64, error) {
+ task, _, err := c.giteaClient.MigrateRepo(gitea.MigrateRepoOption{
+ RepoName: req.Name,
+ RepoOwner: req.Namespace,
+ CloneAddr: req.CloneUrl,
+ Service: gitea.GitServicePlain,
+ AuthUsername: req.Username,
+ AuthPassword: req.AccessToken,
+ Mirror: true,
+ Private: req.Private,
+ Description: req.Description,
+ Wiki: false,
+ Milestones: false,
+ Labels: false,
+ Issues: false,
+ PullRequests: false,
+ Releases: false,
+ LFS: req.SyncLfs,
+ })
+ if err != nil {
+ return 0, err
+ }
+ return task.ID, nil
+}
+
+func (c *MirrorClient) GetMirrorTaskInfo(ctx context.Context, taskId int64) (*mirrorserver.MirrorTaskInfo, error) {
+ ts, _, err := c.giteaClient.GetUserTaskInfo(taskId)
+ if err != nil {
+ return nil, err
+ }
+
+ mti := &mirrorserver.MirrorTaskInfo{
+ Status: mirrorserver.TaskStatus(ts.Status),
+ Message: ts.Message,
+ RepoID: ts.RepoID,
+ RepoName: ts.RepoName,
+ StartedAt: ts.StartedAt,
+ EndedAt: ts.EndedAt,
+ }
+ return mti, nil
+}
+
+func (c *MirrorClient) CreatePushMirror(ctx context.Context, req mirrorserver.CreatePushMirrorReq) error {
+ _, err := c.giteaClient.CreatePushMirror(MirrorServerNamespace, req.Name, gitea.CreatePushMirrorOption{
+ RemoteAddress: req.PushUrl,
+ RemoteUsername: req.Username,
+ RemotePassword: req.AccessToken,
+ Interval: "8h",
+ SyncOnCommit: true,
+ })
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *MirrorClient) MirrorSync(ctx context.Context, req mirrorserver.MirrorSyncReq) error {
+ _, err := c.giteaClient.MirrorSync(req.Namespace, req.Name)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+
+
package gitea
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+
+ "github.com/OpenCSGs/gitea-go-sdk/gitea"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+)
+
+type MirrorClient struct {
+ giteaClient *gitea.Client
+ config *config.Config
+}
+
+type TokenResponse struct {
+ SHA1 string `json:"sha1"`
+ Message string `json:"message"`
+}
+
+func NewMirrorClient(config *config.Config) (client *MirrorClient, err error) {
+ ctx := context.Background()
+ token, err := findOrCreateAccessToken(ctx, config)
+ if err != nil {
+ slog.Error("Failed to find or create token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+ giteaClient, err := gitea.NewClient(
+ config.MirrorServer.Host,
+ gitea.SetContext(ctx),
+ gitea.SetToken(token.Token),
+ gitea.SetBasicAuth(config.MirrorServer.Username, config.MirrorServer.Password),
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ return &MirrorClient{giteaClient: giteaClient, config: config}, nil
+}
+
+func findOrCreateAccessToken(ctx context.Context, config *config.Config) (*database.GitServerAccessToken, error) {
+ gs := database.NewGitServerAccessTokenStore()
+ tokens, err := gs.FindByType(ctx, "mirror")
+ if err != nil {
+ slog.Error("Fail to get mirror server access token from database", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+
+ if len(tokens) == 0 {
+ access_token, err := generateAccessTokenFromGitea(config)
+ if err != nil {
+ slog.Error("Fail to create mirror server access token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+ gToken := &database.GitServerAccessToken{
+ Token: access_token,
+ ServerType: database.MirrorServer,
+ }
+
+ gToken, err = gs.Create(ctx, gToken)
+ if err != nil {
+ slog.Error("Fail to create mirror server access token", slog.String("error: ", err.Error()))
+ return nil, err
+ }
+
+ return gToken, nil
+ }
+ return &tokens[0], nil
+}
+
+func encodeCredentials(username, password string) string {
+ credentials := username + ":" + password
+ return base64.StdEncoding.EncodeToString([]byte(credentials))
+}
+
+func generateAccessTokenFromGitea(config *config.Config) (string, error) {
+ username := config.MirrorServer.Username
+ password := config.MirrorServer.Password
+ giteaUrl := fmt.Sprintf("%s/api/v1/users/%s/tokens", config.MirrorServer.Host, username)
+ authHeader := encodeCredentials(username, password)
+ data := map[string]any{
+ "name": "mirror_access_token",
+ "scopes": []string{
+ "write:user",
+ "write:admin",
+ "write:organization",
+ "write:repository",
+ },
+ }
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ slog.Error("Error encoding JSON data:", "error", err)
+ return "", err
+ }
+
+ req, err := http.NewRequest("POST", giteaUrl, bytes.NewBuffer(jsonData))
+ if err != nil {
+ slog.Error("Error creating request:", "error", err)
+ return "", err
+ }
+
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Basic "+authHeader)
+
+ client := http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ slog.Error("Error sending request:", "error", err)
+ return "", err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ slog.Error("Error reading response body:", "error", err)
+ return "", err
+ }
+
+ var tokenResponse TokenResponse
+ err = json.Unmarshal(body, &tokenResponse)
+ if err != nil {
+ slog.Error("Error decoding JSON response:", "error", err)
+ return "", err
+ }
+
+ if tokenResponse.Message != "" {
+ return "", errors.New(tokenResponse.Message)
+ }
+
+ return tokenResponse.SHA1, nil
+}
+
+
+
package llm
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type Client struct {
+ client *http.Client
+}
+
+func NewClient() *Client {
+ return &Client{
+ client: http.DefaultClient,
+ }
+}
+
+func (c *Client) Chat(ctx context.Context, endpoint string, headers map[string]string, data types.LLMReqBody) (<-chan string, error) {
+ slog.Debug("chat with llm", slog.Any("endpoint", endpoint), slog.Any("data", data))
+ rc, err := c.doSteamRequest(ctx, http.MethodPost, endpoint, headers, data)
+ if err != nil {
+ return nil, fmt.Errorf("do llm stream request, error: %w", err)
+ }
+
+ return c.readToChannel(rc), nil
+}
+
+func (c *Client) doSteamRequest(ctx context.Context, method, url string, headers map[string]string, data interface{}) (io.ReadCloser, error) {
+ var buf io.Reader
+ if data != nil {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ buf = bytes.NewBuffer(jsonData)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, url, buf)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Connection", "keep-alive")
+
+ for k, v := range headers {
+ req.Header.Set(k, v)
+ }
+
+ resp, err := c.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return nil, fmt.Errorf("unexpected http status code:%d", resp.StatusCode)
+ }
+
+ return resp.Body, nil
+}
+
+func (c *Client) readToChannel(rc io.ReadCloser) <-chan string {
+ output := make(chan string, 2)
+ br := bufio.NewReader(rc)
+
+ go func() {
+ for {
+ line, _, err := br.ReadLine()
+ if err != nil {
+ slog.Warn("remote reader aborted", slog.Any("error", err))
+ rc.Close()
+ close(output)
+ break
+ }
+ if len(line) > 0 {
+ output <- string(line)
+ }
+ }
+ }()
+
+ return output
+}
+
+
+
package multisync
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type Client interface {
+ Latest(ctx context.Context, currentVersion int64) (types.SyncVersionResponse, error)
+ ModelInfo(ctx context.Context, v types.SyncVersion) (*types.Model, error)
+ DatasetInfo(ctx context.Context, v types.SyncVersion) (*types.Dataset, error)
+ ReadMeData(ctx context.Context, v types.SyncVersion) (string, error)
+ FileList(ctx context.Context, v types.SyncVersion) ([]types.File, error)
+}
+
+func FromOpenCSG(endpoint string, accessToken string) Client {
+ return &commonClient{
+ endpoint: endpoint,
+ hc: http.DefaultClient,
+ authToken: accessToken,
+ }
+}
+
+type commonClient struct {
+ endpoint string
+ hc *http.Client
+ authToken string
+}
+
+func (c *commonClient) Latest(ctx context.Context, currentVersion int64) (types.SyncVersionResponse, error) {
+ url := fmt.Sprintf("%s/api/v1/sync/version/latest?cur=%d", c.endpoint, currentVersion)
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ req.Header.Add("Authorization", "Bearer "+c.authToken)
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return types.SyncVersionResponse{}, fmt.Errorf("failed to get latest version from endpoint %s, param cur:%d, cause: %w",
+ c.endpoint, currentVersion, err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ var data bytes.Buffer
+ _, _ = data.ReadFrom(resp.Body)
+ return types.SyncVersionResponse{}, fmt.Errorf("failed to get latest version from endpoint %s, param cur:%d, status code: %d, body: %s",
+ c.endpoint, currentVersion, resp.StatusCode, data.String())
+ }
+ var svc types.SyncVersionResponse
+ err = json.NewDecoder(resp.Body).Decode(&svc)
+ if err != nil {
+ return types.SyncVersionResponse{}, fmt.Errorf("failed to decode response body as types.SyncVersionResponse, cause: %w", err)
+ }
+ return svc, nil
+}
+
+func (c *commonClient) ModelInfo(ctx context.Context, v types.SyncVersion) (*types.Model, error) {
+ namespace, name, _ := strings.Cut(v.RepoPath, "/")
+ url := fmt.Sprintf("%s/api/v1/models/%s/%s", c.endpoint, namespace, name)
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ req.Header.Add("Authorization", "Bearer "+c.authToken)
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get model info from endpoint %s, repo path:%s, cause: %w",
+ c.endpoint, v.RepoPath, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to get model info from endpoint %s, repo path:%s, status code: %d",
+ c.endpoint, v.RepoPath, resp.StatusCode)
+ }
+ var res types.ModelResponse
+ err = json.NewDecoder(resp.Body).Decode(&res)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode response body as types.Model, cause: %w", err)
+ }
+ return &res.Data, nil
+}
+
+func (c *commonClient) DatasetInfo(ctx context.Context, v types.SyncVersion) (*types.Dataset, error) {
+ namespace, name, _ := strings.Cut(v.RepoPath, "/")
+ url := fmt.Sprintf("%s/api/v1/datasets/%s/%s", c.endpoint, namespace, name)
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ req.Header.Add("Authorization", "Bearer "+c.authToken)
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get dataset info from endpoint %s, repo path:%s, cause: %w",
+ c.endpoint, v.RepoPath, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to get dataset info from endpoint %s, repo path:%s, status code: %d",
+ c.endpoint, v.RepoPath, resp.StatusCode)
+ }
+ var res types.DatasetResponse
+ err = json.NewDecoder(resp.Body).Decode(&res)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode response body as types.Dataset, cause: %w", err)
+ }
+ return &res.Data, nil
+}
+
+func (c *commonClient) ReadMeData(ctx context.Context, v types.SyncVersion) (string, error) {
+ namespace, name, _ := strings.Cut(v.RepoPath, "/")
+ url := fmt.Sprintf("%s/api/v1/%ss/%s/%s/raw/README.md", c.endpoint, v.RepoType, namespace, name)
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ req.Header.Add("Authorization", "Bearer "+c.authToken)
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return "", fmt.Errorf("failed to get readme data endpoint %s, repo path:%s, cause: %w",
+ c.endpoint, v.RepoPath, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return "", fmt.Errorf("failed to get readme data from endpoint %s, repo path:%s, status code: %d",
+ c.endpoint, v.RepoPath, resp.StatusCode)
+ }
+ var res types.ReadMeResponse
+ err = json.NewDecoder(resp.Body).Decode(&res)
+ if err != nil {
+ return "", fmt.Errorf("failed to decode response body as types.Dataset, cause: %w", err)
+ }
+ return res.Data, nil
+}
+
+func (c *commonClient) FileList(ctx context.Context, v types.SyncVersion) ([]types.File, error) {
+ namespace, name, _ := strings.Cut(v.RepoPath, "/")
+ url := fmt.Sprintf("%s/api/v1/%ss/%s/%s/all_files", c.endpoint, v.RepoType, namespace, name)
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ req.Header.Add("Authorization", "Bearer "+c.authToken)
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get readme data endpoint %s, repo path:%s, cause: %w",
+ c.endpoint, v.RepoPath, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return nil, fmt.Errorf("failed to get readme data from endpoint %s, repo path:%s, status code: %d",
+ c.endpoint, v.RepoPath, resp.StatusCode)
+ }
+ var res types.AllFilesResponse
+ err = json.NewDecoder(resp.Body).Decode(&res)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode response body as types.Dataset, cause: %w", err)
+ }
+ return res.Data, nil
+}
+
+
+
package parquet
+
+import (
+ "database/sql"
+ "fmt"
+ "log/slog"
+ "strings"
+
+ _ "github.com/marcboeker/go-duckdb"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type Reader interface {
+ RowCount(objNames []string, req types.QueryReq) (count int, err error)
+ TopN(objName string, count int) (columns []string, columnsType []string, rows [][]interface{}, err error)
+ FetchRows(objNames []string, req types.QueryReq) (columns []string, columnsType []string, rows [][]interface{}, err error)
+}
+
+type duckdbReader struct {
+ db *sql.DB
+ bucket string
+}
+
+// NewS3Reader create a new reader to read from s3 compatible object storage service
+func NewS3Reader(cfg *config.Config) (Reader, error) {
+ s3SetupSql := fmt.Sprintf(`
+ INSTALL httpfs;
+ LOAD httpfs;
+ SET s3_region = '%s';
+ SET s3_endpoint = '%s';
+ SET s3_url_style = 'vhost';
+ SET s3_access_key_id = '%s';
+ SET s3_secret_access_key = '%s';
+ SET s3_use_ssl = %t;
+ `, cfg.S3.Region, cfg.S3.Endpoint, cfg.S3.AccessKeyID, cfg.S3.AccessKeySecret, cfg.S3.EnableSSL)
+ db, err := sql.Open("duckdb", "")
+ if err != nil {
+ return nil, fmt.Errorf("failed to connect to duckdb, cause:%w", err)
+ }
+ slog.Debug("setup duckdb", slog.String("sql", s3SetupSql))
+ _, err = db.Exec(s3SetupSql)
+ if err != nil {
+ return nil, fmt.Errorf("failed to setup s3 for duckdb, cause:%w", err)
+ }
+ slog.Info("setup duckdb succeeded")
+
+ return &duckdbReader{db: db, bucket: cfg.S3.Bucket}, nil
+}
+
+// RowCount returns the total number of rows in a parquet file in S3 bucket.
+func (r *duckdbReader) RowCount(objNames []string, req types.QueryReq) (int, error) {
+ multiFiles := r.genSelectMultiObjStr(objNames)
+ whereStr := ""
+ if len(req.Where) > 0 {
+ whereStr = fmt.Sprintf("WHERE %s", req.Where)
+ }
+ selectCount := fmt.Sprintf("SELECT count(*) FROM read_parquet(%s, union_by_name = true) %s;", multiFiles, whereStr)
+ row := r.db.QueryRow(selectCount)
+ if row.Err() != nil {
+ return 0, fmt.Errorf("failed to get row count: %w", row.Err())
+ }
+ var count int
+ err := row.Scan(&count)
+ return count, err
+}
+
+// TopN returns the top N rows of a parquet file in S3 bucket.
+func (r *duckdbReader) TopN(objName string, count int) ([]string, []string, [][]interface{}, error) {
+ topN := fmt.Sprintf("SELECT * FROM read_parquet('s3://%s/%s') limit %d;", r.bucket, objName, count)
+ slog.Debug("query topN", slog.String("query", topN))
+ rows, err := r.db.Query(topN)
+ if err != nil {
+ return nil, nil, nil, fmt.Errorf("failed to execute query,cause:%w", err)
+ }
+ defer rows.Close()
+ return r.getColumnsAndValues(rows)
+}
+
+func (r *duckdbReader) FetchRows(objNames []string, req types.QueryReq) ([]string, []string, [][]interface{}, error) {
+ multiFiles := r.genSelectMultiObjStr(objNames)
+ offset := (req.PageIndex - 1) * req.PageSize
+ whereStr := ""
+ if len(req.Where) > 0 {
+ whereStr = fmt.Sprintf("WHERE %s", req.Where)
+ }
+ orderbyStr := ""
+ if len(strings.Trim(req.Orderby, " ")) > 0 {
+ orderbyStr = "order by " + req.Orderby
+ }
+ querySql := fmt.Sprintf("SELECT * FROM read_parquet(%s, union_by_name = true) %s %s limit %d offset %d;", multiFiles, whereStr, orderbyStr, req.PageSize, offset)
+ rows, err := r.db.Query(querySql)
+ if err != nil {
+ return nil, nil, nil, fmt.Errorf("failed to execute fetch rows, cause:%w", err)
+ }
+ defer rows.Close()
+ return r.getColumnsAndValues(rows)
+}
+
+func (r *duckdbReader) getColumnsAndValues(rows *sql.Rows) ([]string, []string, [][]interface{}, error) {
+ colsType, err := rows.ColumnTypes()
+ if err != nil {
+ return nil, nil, nil, fmt.Errorf("failed to get duckdb query columns type, cause:%w", err)
+ }
+
+ var (
+ columns []string
+ columnsType []string
+ )
+ for _, columnType := range colsType {
+ columns = append(columns, columnType.Name())
+ columnsType = append(columnsType, columnType.ScanType().Name())
+ }
+ values := make([][]interface{}, 0)
+
+ for rows.Next() {
+ fields := make([]interface{}, len(columns))
+ pointers := make([]interface{}, len(columns))
+ for i := range fields {
+ pointers[i] = &fields[i]
+ }
+ // Scan values into the slice
+ if err := rows.Scan(pointers...); err != nil {
+ return nil, nil, nil, fmt.Errorf("failed to scan rows, cause:%w", err)
+ // log.Fatal(err)
+ }
+ values = append(values, fields)
+ }
+ return columns, columnsType, values, nil
+}
+
+func (r *duckdbReader) genSelectMultiObjStr(objNames []string) string {
+ var parquetFiles []string
+ for _, objName := range objNames {
+ parquetFiles = append(parquetFiles, fmt.Sprintf("'s3://%s/%s'", r.bucket, objName))
+ }
+ fileStr := ""
+ if len(objNames) > 1 {
+ fileStr = "[" + strings.Join(parquetFiles, ",") + "]"
+ } else {
+ fileStr = parquetFiles[0]
+ }
+
+ return fileStr
+}
+
+
+
package proxy
+
+import (
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+)
+
+type ReverseProxy struct {
+ target *url.URL
+}
+
+func NewReverseProxy(target string) (*ReverseProxy, error) {
+ url, err := url.Parse(target)
+ if err != nil {
+ return nil, err
+ }
+ return &ReverseProxy{
+ target: url,
+ }, nil
+}
+
+func (rp *ReverseProxy) ServeHTTP(w http.ResponseWriter, r *http.Request, api string) {
+ proxy := httputil.NewSingleHostReverseProxy(rp.target)
+ proxy.Director = func(req *http.Request) {
+ req.Host = rp.target.Host
+ req.URL.Host = rp.target.Host
+ req.URL.Scheme = rp.target.Scheme
+ if len(api) > 0 {
+ // change url to given api
+ req.URL.Path = api
+ }
+
+ // debug only
+ // {
+ // slog.Info("request of redirector", slog.Any("req", *req.URL))
+ // data, _ := httputil.DumpRequestOut(req, false)
+ // fmt.Println(string(data))
+ // }
+ }
+ proxy.ModifyResponse = func(r *http.Response) error {
+ // data, err := httputil.DumpResponse(r, true)
+ // fmt.Println(string(data))
+ // remove duplicated header
+ r.Header.Del("Access-Control-Allow-Credentials")
+ r.Header.Del("Access-Control-Allow-Headers")
+ r.Header.Del("Access-Control-Allow-Methods")
+ r.Header.Del("Access-Control-Allow-Origin")
+ return nil
+ }
+ proxy.ServeHTTP(w, r)
+}
+
+
+
package rpc
+
+import "net/http"
+
+type RequestOption interface {
+ Set(req *http.Request)
+}
+
+type authWithApiKey struct {
+ apiKey string
+}
+
+func AuthWithApiKey(apiKey string) RequestOption {
+ return authWithApiKey{apiKey: apiKey}
+}
+
+func (a authWithApiKey) Set(req *http.Request) {
+ req.Header.Set("Authorization", "Bearer "+a.apiKey)
+}
+
+
+
package rpc
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+)
+
+func NewHttpClient(endpoint string, opts ...RequestOption) *HttpClient {
+ return &HttpClient{
+ endpoint: endpoint,
+ hc: http.DefaultClient,
+ authOpts: opts,
+ }
+}
+
+type HttpClient struct {
+ endpoint string
+ hc *http.Client
+ authOpts []RequestOption
+}
+
+func (c *HttpClient) Get(ctx context.Context, path string, outObj interface{}) error {
+ path = fmt.Sprintf("%s%s", c.endpoint, path)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, path, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+ for _, opt := range c.authOpts {
+ opt.Set(req)
+ }
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to do http request, path:%s, err:%w", path, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to get response, path:%s, status:%d", path, resp.StatusCode)
+ }
+ return json.NewDecoder(resp.Body).Decode(outObj)
+}
+
+func (c *HttpClient) Post(ctx context.Context, path string, data interface{}, outObj interface{}) error {
+ path = fmt.Sprintf("%s%s", c.endpoint, path)
+ // serialize data as http request body
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return fmt.Errorf("failed to marshal data: %w", err)
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewBuffer(jsonData))
+ if err != nil {
+ return fmt.Errorf("failed to create request: %w", err)
+ }
+ for _, opt := range c.authOpts {
+ opt.Set(req)
+ }
+ resp, err := c.hc.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to do http request, path:%s, err:%w", path, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to get response, path:%s, status:%d", path, resp.StatusCode)
+ }
+ return json.NewDecoder(resp.Body).Decode(outObj)
+}
+
+
+
package rpc
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type ModerationSvcClient interface {
+ PassTextCheck(ctx context.Context, scenario, text string) (*CheckResult, error)
+ PassImageCheck(ctx context.Context, scenario, ossBucketName, ossObjectName string) (*CheckResult, error)
+ SubmitRepoCheck(ctx context.Context, repoType types.RepositoryType, namespace, name string) error
+}
+
+type CheckResult struct {
+ IsSensitive bool `json:"is_sensitive"`
+ Reason string `json:"reason"`
+}
+
+type ModerationSvcHttpClient struct {
+ hc *HttpClient
+}
+
+func NewModerationSvcHttpClient(endpoint string, opts ...RequestOption) ModerationSvcClient {
+ return &ModerationSvcHttpClient{
+ hc: NewHttpClient(endpoint, opts...),
+ }
+}
+
+func (c *ModerationSvcHttpClient) PassTextCheck(ctx context.Context, scenario, text string) (*CheckResult, error) {
+ type CheckRequest struct {
+ Scenario string `json:"scenario"`
+ Text string `json:"text"`
+ }
+
+ req := &CheckRequest{
+ Scenario: scenario,
+ Text: text,
+ }
+ const path = "/api/v1/text"
+ var resp httpbase.R
+ resp.Data = &CheckResult{}
+ err := c.hc.Post(ctx, path, req, &resp)
+ if err != nil {
+ return nil, err
+ }
+ return resp.Data.(*CheckResult), nil
+}
+
+func (c *ModerationSvcHttpClient) PassImageCheck(ctx context.Context, scenario, ossBucketName, ossObjectName string) (*CheckResult, error) {
+ type CheckRequest struct {
+ Scenario string `json:"scenario"`
+ OssBucketName string `json:"oss_bucket_name"`
+ OssObjectName string `json:"oss_object_name"`
+ }
+
+ req := &CheckRequest{
+ Scenario: scenario,
+ OssBucketName: ossBucketName,
+ OssObjectName: ossObjectName,
+ }
+ var resp httpbase.R
+ resp.Data = &CheckResult{}
+ const path = "/api/v1/image"
+ err := c.hc.Post(ctx, path, req, &resp)
+ if err != nil {
+ return nil, err
+ }
+ return resp.Data.(*CheckResult), nil
+}
+
+func (c *ModerationSvcHttpClient) SubmitRepoCheck(ctx context.Context, repoType types.RepositoryType, namespace, name string) error {
+ type CheckRequest struct {
+ RepoType types.RepositoryType `json:"repo_type"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ }
+
+ req := &CheckRequest{
+ RepoType: repoType,
+ Namespace: namespace,
+ Name: name,
+ }
+ const path = "/api/v1/repo"
+ var resp httpbase.R
+ return c.hc.Post(ctx, path, req, &resp)
+}
+
+
+
package rpc
+
+import (
+ "context"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+)
+
+type PaymentSvcClient interface {
+ CreateSimplePayment(ctx context.Context,
+ orderNo string,
+ subject string,
+ body string,
+ amount *money.Money,
+ extra string,
+ channel consts.PaymentChannel) (*PaymentResponse, error)
+}
+
+type PaymentSvcHttpClient struct {
+ hc *HttpClient
+}
+
+func NewPaymentSvcHttpClient(endpoint string, opts ...RequestOption) PaymentSvcClient {
+ return &PaymentSvcHttpClient{
+ hc: NewHttpClient(endpoint, opts...),
+ }
+}
+
+func (c *PaymentSvcHttpClient) CreateSimplePayment(
+ ctx context.Context,
+ orderNo string,
+ subject string,
+ body string,
+ amount *money.Money,
+ extra string,
+ channel consts.PaymentChannel) (*PaymentResponse, error) {
+ yuanAmount, _ := amount.ToYuanFloat()
+
+ req := &types.CreatePaymentReq{
+ OrderNo: orderNo,
+ Amount: yuanAmount,
+ Channel: channel,
+ Subject: subject,
+ Body: body,
+ Extra: extra,
+ }
+
+ url := "/api/v1/payment/simple-payments"
+ resp := &PaymentResponse{}
+ err := c.hc.Post(ctx, url, req, resp)
+ if err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
+
+
+
package rpc
+
+import (
+ "context"
+ "fmt"
+
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/git/membership"
+)
+
+type UserSvcClient interface {
+ GetMemberRole(ctx context.Context, orgName, userName string) (membership.Role, error)
+ GetNameSpaceInfo(ctx context.Context, path string) (*Namespace, error)
+ GetUserInfo(ctx context.Context, userName, visitorName string) (*User, error)
+ GetOrCreateFirstAvaiTokens(ctx context.Context, userName, visitorName, app, tokenName string) (string, error)
+}
+
+//go:generate mockgen -destination=mocks/client.go -package=mocks . Client
+
+type UserSvcHttpClient struct {
+ hc *HttpClient
+}
+
+func NewUserSvcHttpClient(endpoint string, opts ...RequestOption) UserSvcClient {
+ return &UserSvcHttpClient{
+ hc: NewHttpClient(endpoint, opts...),
+ }
+}
+
+func (c *UserSvcHttpClient) GetMemberRole(ctx context.Context, orgName, userName string) (membership.Role, error) {
+ // write code to call user service api "/api/v1/organization/{orgName}/members/{userName}"
+ url := fmt.Sprintf("/api/v1/organization/%s/members/%s?current_user=%s", orgName, userName, userName)
+ var r httpbase.R
+ r.Data = membership.RoleUnknown
+ err := c.hc.Get(ctx, url, &r)
+ if err != nil {
+ return membership.RoleUnknown, fmt.Errorf("failed to get member role: %w", err)
+ }
+
+ role, ok := r.Data.(string)
+ if !ok {
+ return membership.RoleUnknown, fmt.Errorf("failed to convert r.Data '%v' to membership.Role", r.Data)
+ }
+ return membership.Role(role), nil
+}
+
+func (c *UserSvcHttpClient) GetNameSpaceInfo(ctx context.Context, path string) (*Namespace, error) {
+ // write code to call user service api "/api/v1/namespace/{path}"
+ url := fmt.Sprintf("/api/v1/namespace/%s", path)
+ var r httpbase.R
+ r.Data = &Namespace{}
+ err := c.hc.Get(ctx, url, &r)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace '%s' info: %w", path, err)
+ }
+
+ return r.Data.(*Namespace), nil
+}
+
+func (c *UserSvcHttpClient) GetUserInfo(ctx context.Context, userName, visitorName string) (*User, error) {
+ url := fmt.Sprintf("/api/v1/user/%s?current_user=%s", userName, visitorName)
+ var r httpbase.R
+ r.Data = &User{}
+ err := c.hc.Get(ctx, url, &r)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user '%s' info: %w", userName, err)
+ }
+
+ return r.Data.(*User), nil
+}
+
+func (c *UserSvcHttpClient) GetOrCreateFirstAvaiTokens(ctx context.Context, userName, visitorName, app, tokenName string) (string, error) {
+ url := fmt.Sprintf("/api/v1/user/%s/tokens/first?current_user=%s&app=%s&token_name=%s", userName, visitorName, app, tokenName)
+ var r httpbase.R
+ r.Data = interface{}("")
+ err := c.hc.Get(ctx, url, &r)
+ if err != nil {
+ return "", fmt.Errorf("failed to get user '%s' token for %s: %w", userName, app, err)
+ }
+ return r.Data.(string), nil
+}
+
+
+
package rsa
+
+import (
+ "bytes"
+ "crypto"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/sha256"
+ "crypto/x509"
+ "encoding/base64"
+ "encoding/gob"
+ "encoding/pem"
+ "errors"
+ "fmt"
+ "strings"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+func readPrivateKey(reader KeysReader, fileName string) (*rsa.PrivateKey, error) {
+ block, err := readPEMBlock(reader, fileName)
+ if err != nil {
+ return nil, fmt.Errorf("fail to read PEM block from file %s with error: %w", fileName, err)
+ }
+
+ if block.Type != types.PrivateKeyType {
+ return nil, fmt.Errorf("non-private key PEM block type '%s' from file %s", block.Type, fileName)
+ }
+
+ privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+ if err != nil {
+ return nil, fmt.Errorf("parsing private key with error: %w", err)
+ }
+
+ return privateKey, err
+}
+
+func readPublicKey(reader KeysReader, fileName string) (*rsa.PublicKey, error) {
+ block, err := readPEMBlock(reader, fileName)
+ if err != nil {
+ return nil, fmt.Errorf("fail to read PEM block from file %s with error: %w", fileName, err)
+ }
+
+ if block.Type != types.PublicKeyType {
+ return nil, fmt.Errorf("non-public key PEM block type '%s' from file %s", block.Type, fileName)
+ }
+
+ publicKey, err := x509.ParsePKIXPublicKey(block.Bytes)
+ if err != nil {
+ return nil, fmt.Errorf("parsing public key with error: %w", err)
+ }
+
+ return publicKey.(*rsa.PublicKey), nil
+}
+
+func readPEMBlock(reader KeysReader, fileName string) (*pem.Block, error) {
+ contents, err := reader.ReadKey(fileName)
+ if err != nil {
+ return nil, fmt.Errorf("read file %s with error: %w", fileName, err)
+ }
+
+ block, _ := pem.Decode(contents)
+ if block == nil {
+ return nil, fmt.Errorf("no PEM encoded key found in %s", fileName)
+ }
+ return block, nil
+}
+
+func GenerateData(reader KeysReader, privateKeyfile string, payload types.RSAPayload) (string, error) {
+ var err error
+ var privateKey *rsa.PrivateKey
+
+ privateKey, err = readPrivateKey(reader, privateKeyfile)
+ if err != nil {
+ return "", fmt.Errorf("fail to read private key file %s with error: %w", privateKeyfile, err)
+ }
+
+ gob.Register(types.RSAPayload{})
+
+ payloadBuffer := bytes.Buffer{}
+ payloadEncoder := gob.NewEncoder(&payloadBuffer)
+ if err = payloadEncoder.Encode(&payload); err != nil {
+ return "", fmt.Errorf("fail to encode payload, error: %w", err)
+ }
+
+ hashed := sha256.Sum256(payloadBuffer.Bytes())
+ signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, hashed[:])
+ if err != nil {
+ return "", fmt.Errorf("signing payload with error: %w", err)
+ }
+
+ l := types.RSAInfo{
+ Payload: payloadBuffer.Bytes(),
+ Signature: signature,
+ }
+
+ dataBuffer := bytes.Buffer{}
+ dataEncoder := gob.NewEncoder(&dataBuffer)
+ if err = dataEncoder.Encode(l); err != nil {
+ return "", fmt.Errorf("fail to encode data, error: %w", err)
+ }
+
+ b64 := base64.StdEncoding.EncodeToString(dataBuffer.Bytes())
+
+ result := types.PEMHeader + "\n"
+
+ width := 64
+ for i := 0; ; i += width {
+ if i+width <= len(b64) {
+ result += b64[i:i+width] + "\n"
+ } else {
+ result += b64[i:] + "\n"
+ break
+ }
+ }
+ result += types.PEMFooter
+
+ return result, nil
+}
+
+func VerifyData(reader KeysReader, publicKeyFile, data string) (*types.RSAPayload, error) {
+ var err error
+ var publicKey *rsa.PublicKey
+
+ publicKey, err = readPublicKey(reader, publicKeyFile)
+ if err != nil {
+ return nil, fmt.Errorf("fail to read public key file %s with error: %w", publicKeyFile, err)
+ }
+
+ dataStr := strings.TrimSpace(data)
+
+ if !strings.HasPrefix(dataStr, types.PEMHeader) || !strings.HasSuffix(dataStr, types.PEMFooter) {
+ return nil, errors.New("invalid data in PEM format")
+ }
+
+ base64Str := strings.Replace(dataStr[len(types.PEMHeader):len(dataStr)-len(types.PEMFooter)], "\n", "", -1)
+
+ gob.Register(types.RSAPayload{})
+
+ var rsaInfo types.RSAInfo
+ dataBytes, err := base64.StdEncoding.DecodeString(base64Str)
+ if err != nil {
+ return nil, fmt.Errorf("fail to decode data base64 string with error: %w", err)
+ }
+ dataBuffer := bytes.Buffer{}
+ dataBuffer.Write(dataBytes)
+ dataEncoder := gob.NewDecoder(&dataBuffer)
+ if err = dataEncoder.Decode(&rsaInfo); err != nil {
+ return nil, err
+ }
+
+ hashed := sha256.Sum256(rsaInfo.Payload)
+
+ if err = rsa.VerifyPKCS1v15(publicKey, crypto.SHA256, hashed[:], rsaInfo.Signature); err != nil {
+ return nil, err
+ }
+
+ var payload types.RSAPayload
+ payloadBuffer := bytes.Buffer{}
+ payloadBuffer.Write(rsaInfo.Payload)
+ payloadDecoder := gob.NewDecoder(&payloadBuffer)
+ if err = payloadDecoder.Decode(&payload); err != nil {
+ return nil, err
+ }
+
+ return &payload, nil
+}
+
+
+
package rsa
+
+import (
+ "fmt"
+ "os"
+)
+
+type KeysReader interface {
+ ReadKey(fileName string) ([]byte, error)
+}
+
+type keysReaderImpl struct {
+}
+
+func NewKeysReader() KeysReader {
+ return &keysReaderImpl{}
+}
+
+func (k *keysReaderImpl) ReadKey(fileName string) ([]byte, error) {
+ if _, err := os.Stat(fileName); os.IsNotExist(err) {
+ return nil, fmt.Errorf("file %s does not exist", fileName)
+ }
+
+ contents, err := os.ReadFile(fileName)
+ if err != nil {
+ return nil, fmt.Errorf("read file %s with error: %w", fileName, err)
+ }
+ return contents, nil
+}
+
+
+
package sensitive
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "log/slog"
+ "net/http"
+ "strings"
+
+ openapi "github.com/alibabacloud-go/darabonba-openapi/v2/client"
+ green20220302 "github.com/alibabacloud-go/green-20220302/client"
+ "github.com/alibabacloud-go/tea/tea"
+ "github.com/aliyun/alibaba-cloud-sdk-go/services/green"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+type GreenClient interface {
+ TextScan(request *green.TextScanRequest) (response *TextScanResponse, err error)
+}
+
+type greenClientImpl struct {
+ green *green.Client
+}
+
+func (c *greenClientImpl) TextScan(request *green.TextScanRequest) (response *TextScanResponse, err error) {
+ textScanResponse, err := c.green.TextScan(request)
+ if err != nil {
+ slog.Error("Failed to call TextScan", slog.Any("error", err))
+ return nil, err
+ }
+ data := textScanResponse.GetHttpContentBytes()
+ resp := new(TextScanResponse)
+ err = json.Unmarshal(data, resp)
+ if err != nil {
+ return nil, fmt.Errorf("error unmarshalling scan response: %w", err)
+ }
+ return resp, nil
+}
+
+type Green2022Client interface {
+ GetRegionId() string
+ TextModeration(request *green20220302.TextModerationRequest) (_result *green20220302.TextModerationResponse, _err error)
+ ImageModeration(request *green20220302.ImageModerationRequest) (_result *green20220302.ImageModerationResponse, _err error)
+}
+
+type green2022ClientImpl struct {
+ green *green20220302.Client
+}
+
+func (c *green2022ClientImpl) GetRegionId() string {
+ return tea.StringValue(c.green.RegionId)
+}
+
+func (c *green2022ClientImpl) TextModeration(request *green20220302.TextModerationRequest) (_result *green20220302.TextModerationResponse, _err error) {
+ return c.green.TextModeration(request)
+}
+
+func (c *green2022ClientImpl) ImageModeration(request *green20220302.ImageModerationRequest) (_result *green20220302.ImageModerationResponse, _err error) {
+ return c.green.ImageModeration(request)
+}
+
+/*
+AliyunGreenChecker implements SensitiveChecker by calling Aliyun green sdk
+*/
+type AliyunGreenChecker struct {
+ //improved client
+ green2022 Green2022Client
+ //normal client
+ green GreenClient
+}
+
+func NewAliyunChecker(green GreenClient, green2022 Green2022Client) *AliyunGreenChecker {
+ return &AliyunGreenChecker{
+ green: green,
+ green2022: green2022,
+ }
+}
+
+var _ SensitiveChecker = (*AliyunGreenChecker)(nil)
+
+const smallTextSize = 500
+const LargeTextSize = 9000
+
+// NewAliyunGreenCheckerFromConfig creates a new AliyunGreenChecker
+func NewAliyunGreenCheckerFromConfig(config *config.Config) *AliyunGreenChecker {
+ accessKeyID := config.SensitiveCheck.AccessKeyID
+ accessKeySecret := config.SensitiveCheck.AccessKeySecret
+ region := config.SensitiveCheck.Region
+ slog.Debug("Aliyun client init", slog.String("accessKeyID", accessKeyID),
+ slog.String("accessKeySecret", accessKeySecret),
+ slog.String("region", region))
+
+ aliyunConfig := &openapi.Config{
+ AccessKeyId: tea.String(accessKeyID),
+ AccessKeySecret: tea.String(accessKeySecret),
+ RegionId: tea.String(region),
+ ConnectTimeout: tea.Int(1000),
+ ReadTimeout: tea.Int(2000),
+ }
+ cip, err := green20220302.NewClient(aliyunConfig)
+ if err != nil {
+ log.Fatalf("NewAliyunGreenChecker client enhanced failed: %v", err)
+ }
+
+ c, err := green.NewClientWithAccessKey(region, accessKeyID, accessKeySecret)
+ if err != nil {
+ log.Fatalf("NewAliyunGreenChecker client failed: %v", err)
+ }
+
+ return &AliyunGreenChecker{
+ &green2022ClientImpl{green: cip},
+ &greenClientImpl{green: c},
+ }
+}
+
+// passLargeTextCheck splits large text into smaller `largeTextSize` bytes chunks and check them in batch
+func (c *AliyunGreenChecker) PassLargeTextCheck(ctx context.Context, text string) (*CheckResult, error) {
+ if len(text) > 100*LargeTextSize {
+ return nil, fmt.Errorf("text length can't be greater than 100*%d", LargeTextSize)
+ }
+ tasks := c.SplitTasks(text)
+ content, _ := json.Marshal(
+ map[string]interface{}{
+ "scenes": [...]string{"antispam"},
+ "tasks": tasks,
+ },
+ )
+
+ textScanRequest := green.CreateTextScanRequest()
+ textScanRequest.SetContent(content)
+ resp, err := c.green.TextScan(textScanRequest)
+ if err != nil {
+ slog.Error("Failed to call TextScan", slog.Any("error", err))
+ return nil, err
+ }
+ for _, data := range resp.Data {
+ for _, result := range data.Results {
+ if result.Label == "ad" || result.Label == "flood" {
+ slog.Info("allow ad and flood in text", slog.String("taskId", data.TaskId), slog.String("aliyun_request_id", resp.RequestID))
+ continue
+ }
+
+ if result.Rate < 0.8 {
+ continue
+ }
+
+ if result.Suggestion == "block" {
+ slog.Info("block content", slog.String("content", common.TruncString(data.Content, 128)), slog.String("taskId", data.TaskId),
+ slog.String("aliyun_request_id", resp.RequestID))
+
+ return &CheckResult{IsSensitive: true, Reason: result.Label}, nil
+ }
+ }
+ }
+
+ return &CheckResult{IsSensitive: false}, nil
+}
+
+func (c *AliyunGreenChecker) PassTextCheck(ctx context.Context, scenario Scenario, text string) (*CheckResult, error) {
+ if len(text) > smallTextSize {
+ slog.Info("switch to large text check", slog.String("scenario", string(scenario)), slog.Int("size", len(text)))
+ return c.PassLargeTextCheck(ctx, text)
+ }
+ task := map[string]string{"content": text}
+ serviceParameters, _ := json.Marshal(task)
+ textModerationRequest := &green20220302.TextModerationRequest{
+ Service: tea.String(string(scenario)),
+ ServiceParameters: tea.String(string(serviceParameters)),
+ }
+ resp, err := c.green2022.TextModeration(textModerationRequest)
+ if err != nil {
+ slog.Error("fail to call aliyun TextModeration", slog.String("content", text), slog.Any("error", err))
+ return nil, err
+ }
+
+ if *resp.StatusCode != http.StatusOK || *resp.Body.Code != 200 {
+ slog.Error("aliyun TextModeration return code not 200", slog.String("content", text),
+ slog.String("resp", resp.GoString()))
+ return nil, errors.New(*resp.Body.Message)
+ }
+
+ if len(*resp.Body.Data.Labels) == 0 {
+ return &CheckResult{IsSensitive: false}, nil
+ }
+
+ labelStr := *resp.Body.Data.Labels
+ labels := strings.Split(labelStr, ",")
+ for _, label := range labels {
+ if label == "ad" || label == "flood" {
+ continue
+ }
+
+ slog.Info("sensitive content detected", slog.String("content", text),
+ slog.String("label", label), slog.String("aliyun_request_id", *resp.Body.RequestId))
+ return &CheckResult{IsSensitive: true, Reason: *resp.Body.Data.Reason}, nil
+ }
+
+ return &CheckResult{IsSensitive: false}, nil
+}
+
+func (*AliyunGreenChecker) SplitTasks(text string) []map[string]string {
+ var tasks []map[string]string
+ var i int
+ for i+LargeTextSize < len(text) {
+ tasks = append(tasks, map[string]string{"content": text[i : i+LargeTextSize]})
+ i += LargeTextSize
+ }
+ if i <= len(text) {
+ tasks = append(tasks, map[string]string{"content": text[i:]})
+ }
+ return tasks
+}
+
+func (c *AliyunGreenChecker) PassImageCheck(ctx context.Context, scenario Scenario, ossBucketName, ossObjectName string) (*CheckResult, error) {
+ serviceParameters, _ := json.Marshal(
+ map[string]interface{}{
+ "ossRegionId": c.green2022.GetRegionId(),
+ //for example: my-image-bucket
+ "ossBucketName": ossBucketName,
+ //for example: image/001.jpg
+ "ossObjectName": ossObjectName,
+ },
+ )
+ imageModerationRequest := &green20220302.ImageModerationRequest{
+ Service: tea.String(string(scenario)),
+ ServiceParameters: tea.String(string(serviceParameters)),
+ }
+ resp, err := c.green2022.ImageModeration(imageModerationRequest)
+ if err != nil {
+ slog.Error("fail to call aliyun ImageModeration", slog.String("ossBucketName", ossBucketName),
+ slog.String("ossObjectName", ossObjectName), slog.Any("error", err))
+ return nil, err
+ }
+ slog.Debug("aliyun ImageModeration return", slog.String("resp", resp.GoString()))
+
+ if *resp.StatusCode != http.StatusOK || *resp.Body.Code != 200 {
+ slog.Error("aliyun ImageModeration return code not 200", slog.String("ossBucketName", ossBucketName),
+ slog.String("ossObjectName", ossObjectName),
+ slog.String("resp", resp.GoString()))
+ return nil, errors.New(tea.StringValue(resp.Body.Msg))
+ }
+
+ result := resp.Body.Data.Result
+ //pass check
+ if len(result) == 0 && tea.StringValue(result[0].Label) == "nonLabel" {
+ return &CheckResult{IsSensitive: false}, nil
+ }
+
+ labelMap := make(map[string]float32)
+ for _, r := range result {
+ label, confidence := tea.StringValue(r.Label), tea.Float32Value(r.Confidence)
+ if confidence > 80 {
+ labelMap[label] = confidence
+ }
+ }
+ //pass check
+ if len(labelMap) == 0 {
+ return &CheckResult{IsSensitive: false}, nil
+ }
+
+ slog.Info("sensitive image detected", slog.String("scenario", string(scenario)), slog.String("ossBucketName", ossBucketName),
+ slog.String("ossObjectName", ossObjectName), slog.Any("labels", labelMap), slog.String("aliyun_request_id", *resp.Body.RequestId))
+ // get all the labels in labelMap and join them with ","
+ labels := []string{}
+ for label := range labelMap {
+ labels = append(labels, label)
+ }
+ labelStr := strings.Join(labels, ",")
+ return &CheckResult{IsSensitive: true, Reason: labelStr}, nil
+}
+
+
+
package sensitive
+
+import "context"
+
+type Scenario string
+
+// for text
+const (
+ ScenarioNicknameDetection Scenario = "nickname_detection"
+ ScenarioChatDetection Scenario = "chat_detection"
+ ScenarioCommentDetection Scenario = "comment_detection"
+)
+
+// for image
+const (
+ ScenarioImageProfileCheck Scenario = "profilePhotoCheck"
+ ScenarioImageBaseLineCheck Scenario = "baselineCheck"
+)
+
+func (s Scenario) FromString(scenario string) (Scenario, bool) {
+ switch scenario {
+ case "nickname_detection":
+ return ScenarioNicknameDetection, true
+ case "chat_detection":
+ return ScenarioChatDetection, true
+ case "comment_detection":
+ return ScenarioCommentDetection, true
+ case "profilePhotoCheck":
+ return ScenarioImageProfileCheck, true
+ case "baselineCheck":
+ return ScenarioImageBaseLineCheck, true
+ default:
+ return Scenario(""), false
+ }
+}
+
+type SensitiveChecker interface {
+ PassTextCheck(ctx context.Context, scenario Scenario, text string) (*CheckResult, error)
+ PassImageCheck(ctx context.Context, scenario Scenario, ossBucketName, ossObjectName string) (*CheckResult, error)
+}
+
+type CheckResult struct {
+ IsSensitive bool `json:"is_sensitive"`
+ Reason string `json:"reason"`
+}
+
+
+
package cache
+
+type AccessTokenCache struct {
+ cache *Cache
+}
+
+func NewAccessTokenCache(cache *Cache) *AccessTokenCache {
+ return &AccessTokenCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/redis/go-redis/v9"
+)
+
+type RedisConfig struct {
+ Addr string `comment:"Redis address, e.g. localhost:6379"`
+ Username string `comment:"optional, Redis username"`
+ Password string `comment:"optional, Redis password"`
+ DB int `comment:"optional, Redis DB"`
+}
+
+type Cache struct {
+ core *redis.Client
+ releaseLockScript *redis.Script
+}
+
+func NewCache(ctx context.Context, cfg RedisConfig) (cache *Cache, err error) {
+ const releaseLockScript = `
+local value = redis.call("GET", KEYS[1])
+if not value then
+ return -1 -- not locked
+end
+if value == ARGV[1] then
+ return redis.call("DEL",KEYS[1]) -- lock is successfully released
+else
+ return 0 -- lock does not belongs to us
+end`
+ cache = &Cache{
+ core: redis.NewClient(&redis.Options{
+ Addr: cfg.Addr,
+ Username: cfg.Username,
+ Password: cfg.Password,
+ DB: cfg.DB,
+ }),
+ releaseLockScript: redis.NewScript(releaseLockScript),
+ }
+ err = cache.core.Ping(ctx).Err()
+ if err != nil {
+ err = fmt.Errorf("pinging Redis: %w", err)
+ return
+ }
+ return
+}
+
+func (c *Cache) FlushAll(ctx context.Context) error {
+ return c.core.FlushAll(ctx).Err()
+}
+
+func (c *Cache) ZAdd(ctx context.Context, key string, z redis.Z) error {
+ _, err := c.core.ZAdd(ctx, key, z).Result()
+ return err
+}
+
+func (c *Cache) BZPopMax(ctx context.Context, key string) (*redis.ZWithKey, error) {
+ return c.core.BZPopMax(ctx, time.Second*10, key).Result()
+}
+
+func (c *Cache) Set(ctx context.Context, key string, value string) error {
+ return c.core.Set(ctx, key, value, 0).Err()
+}
+
+func (c *Cache) Get(ctx context.Context, key string) (string, error) {
+ return c.core.Get(ctx, key).Result()
+}
+
+func (c *Cache) Del(ctx context.Context, keys ...string) error {
+ return c.core.Del(ctx, keys...).Err()
+}
+
+func (c *Cache) SAdd(ctx context.Context, key string, members ...interface{}) error {
+ return c.core.SAdd(ctx, key, members...).Err()
+}
+
+func (c *Cache) SIsMember(ctx context.Context, key string, member interface{}) (bool, error) {
+ return c.core.SIsMember(ctx, key, member).Result()
+}
+
+func (c *Cache) SCard(ctx context.Context, key string) (int64, error) {
+ return c.core.SCard(ctx, key).Result()
+}
+
+
+
package cache
+
+type DatasetCache struct {
+ cache *Cache
+}
+
+func NewDatasetCache(cache *Cache) *DatasetCache {
+ return &DatasetCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/google/uuid"
+)
+
+const lockKeyPrefix = "lock-"
+
+var (
+ ErrResourceAlreadyLocked = errors.New("resource is already locked")
+ ErrResourceNotLocked = errors.New("resource is not locked")
+ ErrResourceConflict = errors.New("failed to unlock a resource which is locked by others")
+)
+
+// RunWhileLocked locks the resource, runs user provided callback and finally release the lock.
+// If the lock is not acquirable, this method returns ErrResourceAlreadyLocked immediately.
+//
+// Ref: https://redis.io/docs/manual/patterns/distributed-locks/#correct-implementation-with-a-single-instance
+func (c *Cache) RunWhileLocked(ctx context.Context, resourceName string, expiration time.Duration, fn func(ctx context.Context) error) (err error) {
+ return c.lockAndRun(ctx, resourceName, expiration, false, fn)
+}
+
+// WaitLockToRun wait for the lock to be acquirable and locks the resource, runs user provided callback and finally release the lock.
+// If ctx is canceled or timed out during the waiting, this method returns the error of the ctx.
+func (c *Cache) WaitLockToRun(ctx context.Context, resourceName string, expiration time.Duration, fn func(ctx context.Context) error) (err error) {
+ return c.lockAndRun(ctx, resourceName, expiration, true, fn)
+}
+
+func (c *Cache) lockAndRun(ctx context.Context, resourceName string, expiration time.Duration, waitUntilAvailable bool, fn func(ctx context.Context) error) (err error) {
+ randomStr, err := c.acquireLock(ctx, resourceName, expiration, waitUntilAvailable)
+ if err != nil {
+ err = fmt.Errorf("acquiring lock: %w", err)
+ return
+ }
+
+ defer func() {
+ // ctx may be timed out
+ releasingCtx, cancelRelease := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancelRelease()
+
+ releasingErr := c.releaseLock(releasingCtx, resourceName, randomStr)
+ if releasingErr == nil {
+ return
+ }
+ if err != nil {
+ return
+ }
+ err = fmt.Errorf("releasing lock: %w", releasingErr)
+ }()
+
+ err = fn(ctx)
+ return
+}
+
+// acquireLock implements poor man's resource lock based on single-instance Redis.
+//
+// Ref: https://redis.io/docs/manual/patterns/distributed-locks/#correct-implementation-with-a-single-instance
+func (c *Cache) acquireLock(ctx context.Context, resourceName string, expiration time.Duration, waitUntilAvailable bool) (randomStr string, err error) {
+ if resourceName == "" {
+ err = errors.New("resource name can not be empty")
+ return
+ }
+ key := lockKeyPrefix + resourceName
+
+ randomStr = uuid.New().String()
+
+ ok, err := c.core.SetNX(ctx, key, randomStr, expiration).Result()
+ if err != nil {
+ err = fmt.Errorf("calling redis SETNX: %w", err)
+ return
+ }
+ if ok {
+ return
+ }
+ if !waitUntilAvailable {
+ err = ErrResourceAlreadyLocked
+ return
+ }
+
+ // try to acquire the lock from time to time, until success or time out
+ const (
+ initialInterval = 50 * time.Millisecond
+ maxInterval = 1 * time.Second
+ )
+ var (
+ interval = initialInterval
+ timer = time.NewTimer(interval)
+ )
+ for {
+ select {
+ case <-ctx.Done():
+ // prevent resource leak
+ if !timer.Stop() {
+ <-timer.C
+ }
+
+ err = fmt.Errorf("waiting for lock: %w", ctx.Err())
+ return
+ case <-timer.C:
+ ok, err = c.core.SetNX(ctx, key, randomStr, expiration).Result()
+ if err != nil {
+ err = fmt.Errorf("calling redis SETNX: %w", err)
+ return
+ }
+ if ok {
+ return
+ }
+
+ interval *= 2
+ if interval > maxInterval {
+ interval = maxInterval
+ }
+ timer.Reset(interval)
+ }
+ }
+}
+
+// ReleaseLock releases lock created by AcquireLock.
+//
+// Ref: https://redis.io/docs/manual/patterns/distributed-locks/#correct-implementation-with-a-single-instance
+func (c *Cache) releaseLock(ctx context.Context, resourceName string, providedRandomStr string) (err error) {
+ signal, err := c.releaseLockScript.Run(ctx, c.core, []string{lockKeyPrefix + resourceName}, []interface{}{providedRandomStr}).Int()
+ if err != nil {
+ err = fmt.Errorf("running redis script: %w", err)
+ return
+ }
+
+ switch signal {
+ case -1:
+ err = ErrResourceNotLocked
+ case 0:
+ err = ErrResourceConflict
+ case 1:
+ // relax
+ default:
+ // not reached
+ panic(fmt.Errorf("unexpected signal %d on resourceName %s", signal, resourceName))
+ }
+
+ return
+}
+
+
+
package cache
+
+type MemberCache struct {
+ cache *Cache
+}
+
+func NewMemberCache(cache *Cache) *MemberCache {
+ return &MemberCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type ModelCache struct {
+ cache *Cache
+}
+
+func NewModelCache(cache *Cache) *ModelCache {
+ return &ModelCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type NamespaceCache struct {
+ cache *Cache
+}
+
+func NewNamespaceCache(cache *Cache) *NamespaceCache {
+ return &NamespaceCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type OrgCache struct {
+ cache *Cache
+}
+
+func NewOrgCache(cache *Cache) *OrgCache {
+ return &OrgCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type RepoCache struct {
+ cache *Cache
+}
+
+func NewRepoCache(cache *Cache) *RepoCache {
+ return &RepoCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type SSHKeyCache struct {
+ cache *Cache
+}
+
+func NewSSHKeyCache(cache *Cache) *SSHKeyCache {
+ return &SSHKeyCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type TagCache struct {
+ cache *Cache
+}
+
+func NewTagCache(cache *Cache) *TagCache {
+ return &TagCache{
+ cache: cache,
+ }
+}
+
+
+
package cache
+
+type UserCache struct {
+ cache *Cache
+}
+
+func NewUserCache(cache *Cache) *UserCache {
+ return &UserCache{
+ cache: cache,
+ }
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accessTokenStoreImpl struct {
+ db *DB
+}
+
+type AccessTokenStore interface {
+ Create(ctx context.Context, token *AccessToken) (err error)
+ // Refresh will disable existing access token, and then generate new one
+ Refresh(ctx context.Context, token *AccessToken, newTokenValue string, newExpiredAt time.Time) (*AccessToken, error)
+ FindByID(ctx context.Context, id int64) (token *AccessToken, err error)
+ Delete(ctx context.Context, username, tkName, app string) (err error)
+ IsExist(ctx context.Context, username, tkName, app string) (exists bool, err error)
+ FindByUID(ctx context.Context, uid int64) (token *AccessToken, err error)
+ GetUserGitToken(ctx context.Context, username string) (*AccessToken, error)
+ FindByToken(ctx context.Context, tokenValue, app string) (*AccessToken, error)
+ FindByTokenName(ctx context.Context, username, tokenName, app string) (*AccessToken, error)
+ FindByUser(ctx context.Context, username, app string) ([]AccessToken, error)
+}
+
+func NewAccessTokenStoreWithDB(db *DB) AccessTokenStore {
+ return &accessTokenStoreImpl{db: db}
+}
+
+func NewAccessTokenStore() AccessTokenStore {
+ return &accessTokenStoreImpl{
+ db: defaultDB,
+ }
+}
+
+type AccessToken struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ GitID int64 `bun:",notnull" json:"git_id"`
+ Name string `bun:",notnull" json:"name"`
+ Token string `bun:",notnull" json:"token"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User *User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ //example: csghub, starship
+ Application types.AccessTokenApp `bun:"column:app," json:"application"`
+ Permission string `bun:"," json:"permission"`
+ IsActive bool `bun:",default:true" json:"is_active"`
+ ExpiredAt time.Time `bun:",nullzero" json:"expired_at"`
+ times
+}
+
+func (s *accessTokenStoreImpl) Create(ctx context.Context, token *AccessToken) (err error) {
+ err = s.db.Operator.Core.NewInsert().Model(token).Scan(ctx)
+ return
+}
+
+// Refresh will disable existing access token, and then generate new one
+func (s *accessTokenStoreImpl) Refresh(ctx context.Context, token *AccessToken, newTokenValue string, newExpiredAt time.Time) (*AccessToken, error) {
+ var newToken *AccessToken
+ err := s.db.Core.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, tx bun.Tx) error {
+ _, err := tx.NewUpdate().Model(token).
+ Set("is_active = false").
+ Where("user_id = ? and name = ? and app = ? and (is_active is null or is_active = true) ", token.UserID, token.Name, token.Application).
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to disable old token, err:%w", err)
+ }
+
+ // make a copy
+ newToken = &AccessToken{
+ Name: token.Name,
+ Token: newTokenValue,
+ UserID: token.UserID,
+ Application: token.Application,
+ Permission: token.Permission,
+ IsActive: true,
+ }
+ if newExpiredAt.After(time.Now()) {
+ newToken.ExpiredAt = newExpiredAt
+ } else {
+ //don't change old key's expire time
+ newToken.ExpiredAt = token.ExpiredAt
+ }
+ _, err = tx.NewInsert().Model(newToken).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed insert new token, err:%w", err)
+ }
+
+ return nil
+ })
+
+ newToken.User = token.User
+ return newToken, err
+}
+
+func (s *accessTokenStoreImpl) FindByID(ctx context.Context, id int64) (*AccessToken, error) {
+ var token AccessToken
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&token).
+ Relation("User").
+ Where("access_token.id = ?", id).
+ Scan(ctx)
+ return &token, err
+}
+
+func (s *accessTokenStoreImpl) Delete(ctx context.Context, username, tkName, app string) (err error) {
+ var token AccessToken
+ _, err = s.db.Operator.Core.
+ NewDelete().
+ Model(&token).
+ TableExpr("users AS u").
+ Where("access_token.user_id = u.id").
+ Where("u.username = ?", username).
+ Where("access_token.name = ? and app = ?", tkName, app).
+ Exec(ctx)
+ return
+}
+
+func (s *accessTokenStoreImpl) IsExist(ctx context.Context, username, tkName, app string) (exists bool, err error) {
+ var token AccessToken
+ exists, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&token).
+ Join("JOIN users AS u ON u.id = access_token.user_id").
+ Where("u.username = ?", username).
+ Where("access_token.name = ? and app = ?", tkName, app).
+ Exists(ctx)
+ return
+}
+
+func (s *accessTokenStoreImpl) FindByUID(ctx context.Context, uid int64) (token *AccessToken, err error) {
+ var tokens []AccessToken
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(&tokens).
+ Relation("User").
+ Where("user_id = ?", uid).
+ Where("app = ?", "git").
+ Where("is_active = true and (expired_at is null or expired_at > ?)", time.Now()).
+ Order("created_at DESC").
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if len(tokens) == 0 {
+ return nil, errors.New("access token not found")
+ }
+ token = &tokens[0]
+ return
+}
+
+func (s *accessTokenStoreImpl) GetUserGitToken(ctx context.Context, username string) (*AccessToken, error) {
+ var token AccessToken
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&token).
+ Join("JOIN users AS u ON u.id = access_token.user_id").
+ Where("u.username = ?", username).
+ Where("access_token.app = ?", "git").
+ Where("is_active = true and (access_token.expired_at is null or access_token.expired_at > ?)", time.Now()).
+ Order("created_at DESC").
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &token, nil
+}
+
+func (s *accessTokenStoreImpl) FindByToken(ctx context.Context, tokenValue, app string) (*AccessToken, error) {
+ var token AccessToken
+ q := s.db.Operator.Core.
+ NewSelect().
+ Model(&token).
+ Relation("User").
+ Where("token = ? and is_active = true", tokenValue)
+ if len(app) > 0 {
+ q = q.Where("app = ?", app)
+ }
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &token, nil
+}
+
+func (s *accessTokenStoreImpl) FindByTokenName(ctx context.Context, username, tokenName, app string) (*AccessToken, error) {
+ var token AccessToken
+ q := s.db.Operator.Core.
+ NewSelect().
+ Model(&token).
+ Relation("User").
+ Where("access_token.name = ? and app = ? and is_active = true and username = ?", tokenName, app, username)
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &token, nil
+}
+
+func (s *accessTokenStoreImpl) FindByUser(ctx context.Context, username, app string) ([]AccessToken, error) {
+ var tokens []AccessToken
+ q := s.db.Operator.Core.
+ NewSelect().
+ Model(&tokens).
+ Relation("User").
+ Where("is_active = true and username = ?", username).
+ Order("created_at DESC")
+ if len(app) > 0 {
+ q = q.Where("app = ?", app)
+ }
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return tokens, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "time"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountBillStoreImpl struct {
+ db *DB
+}
+
+type AccountBillStore interface {
+ ListByUserIDAndDate(ctx context.Context, req types.ACCT_BILLS_REQ) (AccountBillRes, error)
+}
+
+func NewAccountBillStore() AccountBillStore {
+ return &accountBillStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountBillStoreWithDB(db *DB) AccountBillStore {
+ return &accountBillStoreImpl{
+ db: db,
+ }
+}
+
+type AccountBill struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ BillDate time.Time `bun:"type:date" json:"bill_date"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ Scene types.SceneType `bun:",notnull" json:"scene"`
+ CustomerID string `bun:",notnull" json:"customer_id"`
+ Value float64 `bun:",notnull" json:"value"`
+ Consumption float64 `bun:",notnull" json:"consumption"`
+ times
+}
+
+type TotalResult struct {
+ TotalValue float64 `bun:"total_value"`
+ TotalConsumption float64 `bun:"total_consumption"`
+}
+
+type AccountBillRes struct {
+ Data []map[string]interface{} `json:"data"`
+ types.ACCT_SUMMARY
+}
+
+func (s *accountBillStoreImpl) ListByUserIDAndDate(ctx context.Context, req types.ACCT_BILLS_REQ) (AccountBillRes, error) {
+ var bill []AccountBill
+ var res []map[string]interface{}
+ q := s.db.Operator.Core.NewSelect().Model(&bill).ColumnExpr("customer_id as instance_name, sum(value) as value, sum(consumption) as consumption").Where("bill_date >= ? and bill_date <= ? and user_uuid = ? and scene = ?", req.StartDate, req.EndDate, req.UserUUID, req.Scene).Group("customer_id")
+
+ count, err := q.Count(ctx)
+ if err != nil {
+ return AccountBillRes{}, err
+ }
+
+ var totalResult TotalResult
+
+ err = s.db.Operator.Core.NewSelect().With("grouped_items", q).TableExpr("grouped_items").ColumnExpr("SUM(value) AS total_value, SUM(consumption) as total_consumption").Scan(ctx, &totalResult)
+ if err != nil {
+ return AccountBillRes{}, err
+ }
+
+ err = q.Order("customer_id").Limit(req.Per).Offset((req.Page-1)*req.Per).Scan(ctx, &res)
+ if err != nil {
+ return AccountBillRes{}, err
+ }
+ return AccountBillRes{
+ Data: res,
+ ACCT_SUMMARY: types.ACCT_SUMMARY{
+ Total: count,
+ TotalValue: totalResult.TotalValue,
+ TotalConsumption: totalResult.TotalConsumption,
+ },
+ }, err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/google/uuid"
+)
+
+type accountEventStoreImpl struct {
+ db *DB
+}
+
+type AccountEventStore interface {
+ GetByEventID(ctx context.Context, eventID uuid.UUID) (*AccountEvent, error)
+ Create(ctx context.Context, input AccountEvent) error
+}
+
+func NewAccountEventStore() AccountEventStore {
+ return &accountEventStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountEventStoreWithDB(db *DB) AccountEventStore {
+ return &accountEventStoreImpl{
+ db: db,
+ }
+}
+
+type AccountEvent struct {
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ EventBody map[string]string `bun:",hstore" json:"event_body"`
+}
+
+func (s *accountEventStoreImpl) GetByEventID(ctx context.Context, eventID uuid.UUID) (*AccountEvent, error) {
+ event := &AccountEvent{}
+ err := s.db.Core.NewSelect().Model(event).Where("event_uuid = ?", eventID).Scan(ctx, event)
+ return event, err
+}
+
+func (s *accountEventStoreImpl) Create(ctx context.Context, input AccountEvent) error {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("insert event log failed, error:%w", err)
+ }
+ return nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountMeteringStoreImpl struct {
+ db *DB
+}
+
+type AccountMeteringStore interface {
+ Create(ctx context.Context, input AccountMetering) error
+ ListByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]AccountMetering, int, error)
+ GetStatByDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]map[string]interface{}, error)
+ ListAllByUserUUID(ctx context.Context, userUUID string) ([]AccountMetering, error)
+}
+
+func NewAccountMeteringStore() AccountMeteringStore {
+ return &accountMeteringStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountMeteringStoreWithDB(db *DB) AccountMeteringStore {
+ return &accountMeteringStoreImpl{
+ db: db,
+ }
+}
+
+type AccountMetering struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ Value float64 `bun:",notnull" json:"value"`
+ ValueType int `bun:",notnull" json:"value_type"`
+ Scene types.SceneType `bun:",notnull" json:"scene"`
+ OpUID string `json:"op_uid"`
+ ResourceID string `bun:",notnull" json:"resource_id"`
+ ResourceName string `bun:",notnull" json:"resource_name"`
+ CustomerID string `json:"customer_id"`
+ RecordedAt time.Time `bun:",notnull" json:"recorded_at"`
+ Extra string `json:"extra"`
+ CreatedAt time.Time `bun:",notnull,default:current_timestamp" json:"created_at"`
+ SkuUnitType string `json:"sku_unit_type"`
+}
+
+func (am *accountMeteringStoreImpl) Create(ctx context.Context, input AccountMetering) error {
+ res, err := am.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("failed to save metering event, error: %w", err)
+ }
+ return nil
+}
+
+func (am *accountMeteringStoreImpl) ListByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]AccountMetering, int, error) {
+ var accountMeters []AccountMetering
+ q := am.db.Operator.Core.NewSelect().Model(&accountMeters).Where("user_uuid = ? and scene = ? and customer_id = ? and recorded_at >= ? and recorded_at <= ?", req.UserUUID, req.Scene, req.InstanceName, req.StartTime, req.EndTime)
+
+ count, err := q.Count(ctx)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to counting recorders, error: %w", err)
+ }
+
+ _, err = q.Order("id DESC").Limit(req.Per).Offset((req.Page-1)*req.Per).Exec(ctx, &accountMeters)
+ if err != nil {
+ return nil, 0, fmt.Errorf("list all meters, error: %w", err)
+ }
+ return accountMeters, count, nil
+}
+
+func (am *accountMeteringStoreImpl) GetStatByDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) ([]map[string]interface{}, error) {
+ var meter []AccountMetering
+ var res []map[string]interface{}
+ err := am.db.Operator.Core.NewSelect().Model(&meter).
+ ColumnExpr("users.username").
+ ColumnExpr("account_metering.user_uuid").
+ ColumnExpr("account_metering.resource_id").
+ ColumnExpr("sum(account_metering.value) as value").
+ Join("join users on users.uuid = account_metering.user_uuid").
+ Where("account_metering.scene = ?", req.Scene).
+ Where("account_metering.recorded_at >= ?", req.StartTime).
+ Where("account_metering.recorded_at <= ?", req.EndTime).
+ Group("users.username").
+ Group("account_metering.user_uuid").
+ Group("account_metering.resource_id").
+ Order("account_metering.resource_id").
+ Order("value desc").
+ Scan(ctx, &res)
+
+ if err != nil {
+ return nil, fmt.Errorf("select metering stat, error: %w", err)
+ }
+ return res, nil
+}
+
+func (am *accountMeteringStoreImpl) ListAllByUserUUID(ctx context.Context, userUUID string) ([]AccountMetering, error) {
+ var accountMeters []AccountMetering
+ err := am.db.Operator.Core.NewSelect().Model(&accountMeters).Where("user_uuid = ?", userUUID).Scan(ctx, &accountMeters)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list all meters by user uuid: %w", err)
+ }
+ return accountMeters, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountOrderStoreImpl struct {
+ db *DB
+}
+
+type AccountOrderStore interface {
+ Create(ctx context.Context, order AccountOrder, statement AccountStatement) error
+ GetByID(ctx context.Context, uuid string) (*AccountOrder, error)
+ GetDetailByID(ctx context.Context, id int64) (*AccountOrderDetail, error)
+}
+
+func NewAccountOrderStore() AccountOrderStore {
+ return &accountOrderStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountOrderStoreWithDB(db *DB) AccountOrderStore {
+ return &accountOrderStoreImpl{
+ db: db,
+ }
+}
+
+type AccountOrder struct {
+ OrderUUID string `bun:",notnull,pk" json:"order_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ OrderStatus types.OrderStatus `bun:",notnull" json:"order_status"`
+ Amount float64 `bun:",notnull" json:"amount"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ EventUUID string `json:"event_uuid"`
+ RecordedAt time.Time `json:"recorded_at"`
+ Details []AccountOrderDetail `bun:"rel:has-many,join:order_uuid=order_uuid" json:"details"`
+}
+
+type AccountOrderDetail struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ OrderUUID string `bun:",notnull" json:"order_uuid"`
+ ResourceID string `bun:",notnull" json:"resource_id"`
+ SkuType types.SKUType `bun:",notnull" json:"sku_type"`
+ SkuKind types.SKUKind `bun:",notnull" json:"sku_kind"`
+ SkuUnitType string `bun:",notnull" json:"sku_unit_type"`
+ OrderCount int `bun:",notnull" json:"order_count"`
+ SkuPriceID int64 `bun:",notnull" json:"sku_price_id"`
+ Amount float64 `bun:",notnull" json:"amount"`
+ BeginTime time.Time `bun:",notnull" json:"begin_time"`
+ EndTime time.Time `bun:",notnull" json:"end_time"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ PresentUUID string `json:"present_uuid"`
+}
+
+func (aos *accountOrderStoreImpl) Create(ctx context.Context, order AccountOrder, input AccountStatement) error {
+ err := aos.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if input.Scene != types.ScenePayOrder {
+ return fmt.Errorf("invalid scene, expect %v, got %v", types.ScenePayOrder, input.Scene)
+ }
+
+ var err error
+
+ err = assertAffectedOneRow(tx.NewInsert().Model(&order).Exec(ctx))
+ if err != nil {
+ return fmt.Errorf("insert order, error:%w", err)
+ }
+
+ for _, detail := range order.Details {
+ err = assertAffectedOneRow(tx.NewInsert().Model(&detail).Exec(ctx))
+ if err != nil {
+ return fmt.Errorf("insert order detail, error:%w", err)
+ }
+ }
+
+ err = DeductAccountFee(ctx, tx, input)
+ if err != nil {
+ return fmt.Errorf("deduct account fee for order, error:%w", err)
+ }
+
+ return nil
+ })
+
+ return err
+}
+
+func (aos *accountOrderStoreImpl) GetByID(ctx context.Context, uuid string) (*AccountOrder, error) {
+ var order AccountOrder
+ err := aos.db.Operator.Core.NewSelect().Model(&order).Where("order_uuid = ?", uuid).Relation("Details").Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select account order by uuid, error:%w", err)
+ }
+ return &order, nil
+}
+
+func (aos *accountOrderStoreImpl) GetDetailByID(ctx context.Context, id int64) (*AccountOrderDetail, error) {
+ var detail AccountOrderDetail
+ err := aos.db.Operator.Core.NewSelect().Model(&detail).Where("id = ?", id).Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select order detail by id, %d, error:%w", id, err)
+ }
+ return &detail, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+)
+
+type accountPresentStoreImpl struct {
+ db *DB
+}
+
+type AccountPresentStore interface {
+ AddPresent(ctx context.Context, input AccountPresent, statement AccountStatement) error
+ FindPresentByUserIDAndScene(ctx context.Context, userID string, activityID int64) (*AccountPresent, error)
+}
+
+func NewAccountPresentStore() AccountPresentStore {
+ return &accountPresentStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountPresentStoreWithDB(db *DB) AccountPresentStore {
+ return &accountPresentStoreImpl{
+ db: db,
+ }
+}
+
+type AccountPresent struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ ActivityID int64 `bun:",notnull" json:"activity_id"`
+ Value float64 `bun:",notnull" json:"value"`
+ OpUID string `bun:",notnull" json:"op_uid"`
+ OpDesc string `bun:",notnull" json:"op_desc"`
+ times
+}
+
+func (ap *accountPresentStoreImpl) AddPresent(ctx context.Context, input AccountPresent, statement AccountStatement) error {
+ err := ap.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&input).Exec(ctx)); err != nil {
+ return fmt.Errorf("insert account present, error:%w", err)
+ }
+
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&statement).Exec(ctx)); err != nil {
+ return fmt.Errorf("insert account statement, error:%w", err)
+ }
+
+ runSql := "update account_users set balance=balance + ? where user_uuid=?"
+ if err := assertAffectedOneRow(tx.Exec(runSql, input.Value, input.UserUUID)); err != nil {
+ return fmt.Errorf("update account balance, error:%w", err)
+ }
+
+ return nil
+ })
+
+ return err
+}
+
+func (ap *accountPresentStoreImpl) FindPresentByUserIDAndScene(ctx context.Context, userID string, activityID int64) (*AccountPresent, error) {
+ present := &AccountPresent{}
+ err := ap.db.Core.NewSelect().Model(present).Where("user_uuid = ? and activity_id = ?", userID, activityID).Limit(1).Scan(ctx, present)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("find account present by user id and activity id, error:%w", err)
+ }
+ return present, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "strconv"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountPriceStoreImpl struct {
+ db *DB
+}
+
+type AccountPriceStore interface {
+ Create(ctx context.Context, input AccountPrice) (*AccountPrice, error)
+ Update(ctx context.Context, input AccountPrice) (*AccountPrice, error)
+ Delete(ctx context.Context, input AccountPrice) error
+ GetByID(ctx context.Context, id int64) (*AccountPrice, error)
+ GetLatestByTime(ctx context.Context, req types.AcctPriceQueryReq) (*AccountPrice, error)
+ ListBySkuType(ctx context.Context, req types.AcctPriceListReq) ([]AccountPrice, int, error)
+}
+
+func NewAccountPriceStore() AccountPriceStore {
+ return &accountPriceStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountPriceStoreWithDB(db *DB) AccountPriceStore {
+ return &accountPriceStoreImpl{
+ db: db,
+ }
+}
+
+type AccountPrice struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ SkuType types.SKUType `bun:",notnull" json:"sku_type"`
+ SkuPrice int64 `bun:",notnull" json:"sku_price"`
+ SkuUnit int64 `bun:",notnull" json:"sku_unit"`
+ SkuDesc string `bun:",notnull" json:"sku_desc"`
+ ResourceID string `bun:",notnull" json:"resource_id"`
+ SkuUnitType string `json:"sku_unit_type"`
+ SkuPriceCurrency string `json:"sku_price_currency"`
+ SkuKind types.SKUKind `json:"sku_kind"`
+ Quota string `json:"quota"`
+ SkuPriceID int64 `json:"sku_price_id"`
+ times
+}
+
+type PriceResp struct {
+ Prices []AccountPrice `json:"data"`
+ Total int `json:"total"`
+}
+
+func (a *accountPriceStoreImpl) Create(ctx context.Context, input AccountPrice) (*AccountPrice, error) {
+ res, err := a.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create price failed,error:%w", err)
+ }
+ return &input, nil
+}
+
+func (a *accountPriceStoreImpl) Update(ctx context.Context, input AccountPrice) (*AccountPrice, error) {
+ _, err := a.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update price, error: %w", err)
+ }
+ return &input, nil
+}
+
+func (a *accountPriceStoreImpl) Delete(ctx context.Context, input AccountPrice) error {
+ _, err := a.db.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to delete price, error: %w", err)
+ }
+ return nil
+}
+
+func (a *accountPriceStoreImpl) GetByID(ctx context.Context, id int64) (*AccountPrice, error) {
+ price := &AccountPrice{}
+ err := a.db.Core.NewSelect().Model(price).Where("id = ?", id).Scan(ctx, price)
+ if err != nil {
+ return nil, fmt.Errorf("select price by id %d, error: %w", id, err)
+ }
+ return price, nil
+}
+
+func (a *accountPriceStoreImpl) GetLatestByTime(ctx context.Context, req types.AcctPriceQueryReq) (*AccountPrice, error) {
+ price := &AccountPrice{}
+ err := a.db.Core.NewSelect().Model(price).
+ Where("sku_type = ?", req.SkuType).
+ Where("sku_kind = ?", req.SkuKind).
+ Where("resource_id = ?", req.ResourceID).
+ Where("sku_unit_type = ?", req.SkuUnitType).
+ Where("created_at <= ?", req.PriceTime).
+ Order("created_at DESC").Limit(1).Scan(ctx, price)
+ if err != nil {
+ return nil, fmt.Errorf("select price by time, error: %w", err)
+ }
+ return price, nil
+}
+
+func (a *accountPriceStoreImpl) ListBySkuType(ctx context.Context, req types.AcctPriceListReq) ([]AccountPrice, int, error) {
+ var result []AccountPrice
+ q := a.db.Core.NewSelect().Model(&result).
+ DistinctOn("sku_type, sku_kind, resource_id, sku_unit_type").
+ Where("sku_type = ?", req.SkuType)
+
+ if len(req.SkuKind) > 0 {
+ skuKindInt, err := strconv.Atoi(req.SkuKind)
+ if err != nil {
+ return nil, 0, fmt.Errorf("invalid sku kind %s, error: %w", req.SkuKind, err)
+ }
+ q.Where("sku_kind = ?", skuKindInt)
+ }
+
+ if len(req.ResourceID) > 0 {
+ q.Where("resource_id = ?", req.ResourceID)
+ }
+
+ count, err := q.Count(ctx)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to counting recorder, error: %w", err)
+ }
+ _, err = q.Order("sku_type ASC").Order("sku_kind ASC").Order("resource_id ASC").
+ Order("sku_unit_type ASC").Order("created_at DESC").
+ Limit(req.Per).Offset((req.Page-1)*req.Per).Exec(ctx, &result)
+ if err != nil {
+ return nil, 0, fmt.Errorf("select prices by type and kind and resource, error: %w", err)
+ }
+ return result, count, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "time"
+)
+
+type accountRechargeStoreImpl struct {
+ db *DB
+}
+
+type AccountRechargeStore interface {
+ CreateRecharge(ctx context.Context, recharge *AccountRecharge) error
+ GetRecharge(ctx context.Context, rechargeUUID string) (*AccountRecharge, error)
+ GetRechargeByOrderNo(ctx context.Context, orderNo string) (*AccountRecharge, error)
+ UpdateRecharge(ctx context.Context, recharge *AccountRecharge) error
+ ListRechargeByUserUUID(ctx context.Context, userUUID string, limit, offset int) ([]*AccountRecharge, error)
+ ListRecharges(ctx context.Context, userUUID string, filter RechargeFilter) ([]*AccountRecharge, error)
+ CountRecharges(ctx context.Context, userUUID string, filter RechargeFilter) (int, error)
+}
+
+func NewAccountRechargeStore() AccountRechargeStore {
+ return &accountRechargeStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountRechargeStoreWithDB(db *DB) AccountRechargeStore {
+ return &accountRechargeStoreImpl{
+ db: db,
+ }
+}
+
+type AccountRecharge struct {
+ RechargeUUID string `bun:",notnull,pk,skipupdate" json:"uuid"` // Recharge object ID
+ OrderNo string `bun:",notnull,unique" json:"order_no"` // Order ID allowed by the payment system
+ UserUUID string `bun:",notnull,skipupdate" json:"user_uuid"` // Target UserUUID for the recharge
+ FromUserUUID string `bun:",notnull,skipupdate" json:"from_user_uuid"` // Source UserUUID for the recharge
+ Amount int64 `bun:",notnull,skipupdate" json:"amount"` // Actual balance received by the user, in cents
+ Currency string `bun:",notnull,skipupdate,default:'CNY'" json:"currency"` // 3-letter ISO currency code in uppercase letters
+ Channel consts.PaymentChannel `bun:",notnull,skipupdate" json:"channel"`
+ PaymentUUID string `bun:",notnull,skipupdate,unique" json:"payment_uuid"`
+ Succeeded bool `json:"succeeded"`
+ Closed bool `json:"closed"`
+ TimeSucceeded time.Time `bun:",nullzero" json:"time_succeeded"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ UpdatedAt time.Time `bun:",notnull,default:current_timestamp" json:"updated_at"`
+ Description string `json:"description"`
+}
+
+func (rs *accountRechargeStoreImpl) CreateRecharge(ctx context.Context, recharge *AccountRecharge) error {
+ _, err := rs.db.Operator.Core.NewInsert().Model(recharge).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("create recharge record, error: %w", err)
+ }
+ return nil
+}
+
+func (rs *accountRechargeStoreImpl) GetRecharge(ctx context.Context, rechargeUUID string) (*AccountRecharge, error) {
+ var recharge AccountRecharge
+ q := rs.db.Operator.Core.NewSelect().
+ Model(&recharge).
+ Where("recharge_uuid = ?", rechargeUUID)
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("get recharge, error: %w", err)
+ }
+ return &recharge, nil
+}
+
+func (rs *accountRechargeStoreImpl) GetRechargeByOrderNo(ctx context.Context, orderNo string) (*AccountRecharge, error) {
+ var recharge AccountRecharge
+ q := rs.db.Operator.Core.NewSelect().
+ Model(&recharge).
+ Where("account_recharge.order_no = ?", orderNo)
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("get recharge, error: %w", err)
+ }
+ return &recharge, nil
+}
+
+func (rs *accountRechargeStoreImpl) UpdateRecharge(ctx context.Context, recharge *AccountRecharge) error {
+ recharge.UpdatedAt = time.Now()
+ _, err := rs.db.Operator.Core.NewUpdate().
+ Model(recharge).
+ Where("recharge_uuid = ?", recharge.RechargeUUID). // 根据 UUID 定位记录
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("update payment record, error: %w", err)
+ }
+ return nil
+}
+
+func (rs *accountRechargeStoreImpl) ListRechargeByUserUUID(ctx context.Context, userUUID string, limit, offset int) ([]*AccountRecharge, error) {
+ var recharges []*AccountRecharge
+ q := rs.db.Operator.Core.NewSelect().
+ Model(&recharges).
+ Where("user_uuid = ?", userUUID).
+ Order("created_at DESC").
+ Limit(limit).
+ Offset(offset)
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list recharges for user %s, error: %w", userUUID, err)
+ }
+ return recharges, nil
+}
+
+func (rs *accountRechargeStoreImpl) applyRechargesFilters(q *bun.SelectQuery, filter RechargeFilter) *bun.SelectQuery {
+
+ if filter.OrderNoPattern != "" {
+ q.Where("account_recharge.order_no LIKE ?", "%"+filter.OrderNoPattern+"%")
+ }
+
+ // Filter by creation date
+ if filter.StartDate != "" {
+ q.Where("account_recharge.created_at >= ?", filter.StartDate)
+ }
+ if filter.EndDate != "" {
+ q.Where("account_recharge.created_at <= ?", filter.EndDate)
+ }
+
+ if filter.Succeeded != nil {
+ q.Where("succeeded = ?", *filter.Succeeded)
+ }
+
+ if filter.Closed != nil {
+ q.Where("closed = ?", *filter.Closed)
+ }
+
+ // Join Payment table and filter by Channel
+ if filter.PaymentChannel != nil {
+ q.Where("channel = ?", *filter.PaymentChannel)
+ }
+
+ return q
+}
+
+func (rs *accountRechargeStoreImpl) ListRecharges(ctx context.Context, userUUID string, filter RechargeFilter) ([]*AccountRecharge, error) {
+ var recharges []*AccountRecharge
+ q := rs.db.Operator.Core.NewSelect().
+ Model(&recharges).
+ Where("user_uuid = ?", userUUID).
+ Order("created_at DESC")
+
+ // Pagination
+ if filter.Limit > 0 {
+ q.Limit(filter.Limit)
+ }
+ if filter.Offset > 0 {
+ q.Offset(filter.Offset)
+ }
+
+ // Apply common filters
+ q = rs.applyRechargesFilters(q, filter)
+
+ err := q.Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list recharges for user %s, error: %w", userUUID, err)
+ }
+ return recharges, nil
+}
+
+func (rs *accountRechargeStoreImpl) CountRecharges(ctx context.Context, userUUID string, filter RechargeFilter) (int, error) {
+ q := rs.db.Operator.Core.NewSelect().
+ Model((*AccountRecharge)(nil)).
+ Where("user_uuid = ?", userUUID)
+
+ // Apply common filters
+ q = rs.applyRechargesFilters(q, filter)
+
+ count, err := q.Count(ctx)
+ if err != nil {
+ return 0, fmt.Errorf("count recharges for user %s, error: %w", userUUID, err)
+ }
+ return count, nil
+}
+
+type RechargeFilter struct {
+ OrderNoPattern string
+ StartDate string
+ EndDate string
+ Succeeded *bool
+ Closed *bool
+ PaymentChannel *consts.PaymentChannel
+ Limit int
+ Offset int
+}
+
+func (f *RechargeFilter) SetOrderNoPattern(pattern string) *RechargeFilter {
+ f.OrderNoPattern = pattern
+ return f
+}
+
+func (f *RechargeFilter) SetDateRange(start, end string) *RechargeFilter {
+ f.StartDate = start
+ f.EndDate = end
+ return f
+}
+
+func (f *RechargeFilter) SetSucceeded(succeeded bool) *RechargeFilter {
+ f.Succeeded = &succeeded
+ return f
+}
+
+func (f *RechargeFilter) SetPaymentChannel(channel consts.PaymentChannel) *RechargeFilter {
+ f.PaymentChannel = &channel
+ return f
+}
+
+func (f *RechargeFilter) SetLimit(limit int) *RechargeFilter {
+ f.Limit = limit
+ return f
+}
+
+func (f *RechargeFilter) SetOffset(offset int) *RechargeFilter {
+ f.Offset = offset
+ return f
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/accounting/utils"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountStatementStoreImpl struct {
+ db *DB
+}
+
+type AccountStatementStore interface {
+ Create(ctx context.Context, input AccountStatement) error
+ ListByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (AccountStatementRes, error)
+ GetByEventID(ctx context.Context, eventID uuid.UUID) (AccountStatement, error)
+}
+
+func NewAccountStatementStore() AccountStatementStore {
+ return &accountStatementStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountStatementStoreWithDB(db *DB) AccountStatementStore {
+ return &accountStatementStoreImpl{
+ db: db,
+ }
+}
+
+type AccountStatement struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ Value float64 `bun:",notnull" json:"value"`
+ Scene types.SceneType `bun:",notnull" json:"scene"`
+ OpUID string `bun:",nullzero" json:"op_uid"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ CustomerID string `json:"customer_id"`
+ EventDate time.Time `bun:"type:date" json:"event_date"`
+ Price float64 `json:"price"`
+ PriceUnit string `json:"price_unit"`
+ Consumption float64 `json:"consumption"`
+ ValueType int `json:"value_type"`
+ ResourceID string `json:"resource_id"`
+ ResourceName string `json:"resource_name"`
+ SkuID int64 `json:"sku_id"`
+ RecordedAt time.Time `json:"recorded_at"`
+ SkuUnit int64 `json:"sku_unit"`
+ SkuUnitType string `json:"sku_unit_type"`
+ SkuPriceCurrency string `json:"sku_price_currency"`
+ BalanceType string `json:"balance_type"`
+ BalanceValue float64 `json:"balance_value"`
+ IsCancel bool `json:"is_cancel"`
+ EventValue float64 `json:"event_value"`
+}
+
+type AccountStatementRes struct {
+ Data []AccountStatement `json:"data"`
+ types.ACCT_SUMMARY
+}
+
+func (as *accountStatementStoreImpl) Create(ctx context.Context, input AccountStatement) error {
+ if input.Scene == types.ScenePortalCharge || input.Scene == types.SceneCashCharge {
+ return as.chargeFeeStatement(ctx, input)
+ } else {
+ return as.deductFeeStatement(ctx, input)
+ }
+}
+
+func (as *accountStatementStoreImpl) chargeFeeStatement(ctx context.Context, input AccountStatement) error {
+ err := as.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if input.Value <= 0 {
+ return fmt.Errorf("charge fee statement value must be positive, got: %f", input.Value)
+ }
+ var err error
+ var acctUser AccountUser
+
+ err = tx.NewSelect().Model(&acctUser).Where("user_uuid = ?", input.UserUUID).Scan(ctx, &acctUser)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ newAcctUser := AccountUser{
+ UserUUID: input.UserUUID,
+ Balance: 0,
+ CashBalance: 0,
+ }
+ res, err := tx.NewInsert().Model(&newAcctUser).Exec(ctx, &newAcctUser)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("insert user account failed, error:%w", err)
+ }
+ } else {
+ return fmt.Errorf("failed to get account user %s, err: %w", input.UserUUID, err)
+ }
+ }
+
+ err = tx.NewSelect().Model(&acctUser).Where("user_uuid = ?", input.UserUUID).For("UPDATE").Scan(ctx, &acctUser)
+ if err != nil {
+ return fmt.Errorf("failed to get account user %s for lock: %w", input.UserUUID, err)
+ }
+
+ err = CheckDuplicatedEvent(ctx, tx, input)
+ if err != nil {
+ if errors.Is(err, types.ErrDuplicatedEvent) {
+ slog.Warn("skip duplicated charge fee by event uuid", slog.Any("input", input))
+ return nil
+ }
+ return fmt.Errorf("check duplicated charge event failed, error: %w", err)
+ }
+
+ runSql := ""
+ if input.Scene == types.SceneCashCharge {
+ input.BalanceType = types.ChargeCashBalance
+ runSql = "update account_users set cash_balance=cash_balance + ? where user_uuid=?"
+ } else {
+ input.BalanceType = types.ChargeBalance
+ runSql = "update account_users set balance=balance + ? where user_uuid=?"
+ }
+
+ err = assertAffectedOneRow(tx.Exec(runSql, input.Value, input.UserUUID))
+ if err != nil {
+ return fmt.Errorf("update %s, error:%w", input.BalanceType, err)
+ }
+
+ acctUser = AccountUser{}
+ err = tx.NewSelect().Model(&acctUser).Where("user_uuid = ?", input.UserUUID).Scan(ctx, &acctUser)
+ if err != nil {
+ return fmt.Errorf("failed to get account user: %w", err)
+ }
+
+ if input.Scene == types.SceneCashCharge {
+ input.BalanceValue = acctUser.CashBalance
+ } else {
+ input.BalanceValue = acctUser.Balance
+ }
+
+ err = assertAffectedOneRow(tx.NewInsert().Model(&input).Exec(ctx))
+ if err != nil {
+ return fmt.Errorf("insert statement, error:%w", err)
+ }
+ return nil
+ })
+
+ return err
+}
+
+func (as *accountStatementStoreImpl) deductFeeStatement(ctx context.Context, input AccountStatement) error {
+ err := as.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if input.Value > 0 {
+ return fmt.Errorf("deduct fee statement value must be negative")
+ }
+
+ var err error
+
+ err = DeductAccountFee(ctx, tx, input)
+ if err != nil {
+ if errors.Is(err, types.ErrDuplicatedEvent) {
+ slog.Warn("skip duplicated deduct fee by event uuid", slog.Any("input", input))
+ return nil
+ }
+ return fmt.Errorf("deduct account fee, error: %w", err)
+ }
+
+ err = updateFeeBill(ctx, tx, input)
+ if err != nil {
+ return fmt.Errorf("update account bill, error: %w", err)
+ }
+
+ return nil
+ })
+
+ return err
+}
+
+func DeductAccountFee(ctx context.Context, tx bun.Tx, input AccountStatement) error {
+ var err error
+ var acctUser AccountUser
+
+ // add lock
+ err = tx.NewSelect().Model(&acctUser).Where("user_uuid = ?", input.UserUUID).For("UPDATE").Scan(ctx, &acctUser)
+ if err != nil {
+ return fmt.Errorf("failed to get account user: %w", err)
+ }
+
+ err = CheckDuplicatedEvent(ctx, tx, input)
+ if err != nil {
+ if errors.Is(err, types.ErrDuplicatedEvent) {
+ // return duplicated error for check
+ return err
+ }
+ return fmt.Errorf("check duplicated order event failed, error: %w", err)
+ }
+
+ remainValue := input.Value
+
+ if remainValue == 0 {
+ // only save statement
+ err = assertAffectedOneRow(tx.NewInsert().Model(&input).Exec(ctx))
+ if err != nil {
+ return fmt.Errorf("insert statement, error:%w", err)
+ }
+ return nil
+ }
+
+ // 1. check and reduce cash
+ if remainValue < 0 && acctUser.CashBalance > 0 {
+ remainValue, err = deductFeeToCashBalance(ctx, tx, input, remainValue, acctUser.CashBalance)
+ if err != nil {
+ return fmt.Errorf("deduct cash balance, error:%w", err)
+ }
+ }
+
+ // 2. check and reduce bonus
+ if remainValue < 0 && acctUser.Balance > 0 {
+ remainValue, err = deductFeeToBalance(ctx, tx, input, remainValue, acctUser.Balance)
+ if err != nil {
+ return fmt.Errorf("deduct balance, error:%w", err)
+ }
+ }
+
+ // 3. check and reduce all in cash
+ if remainValue < 0 {
+ err = tx.NewSelect().Model(&acctUser).Where("user_uuid = ?", input.UserUUID).Scan(ctx, &acctUser)
+ if err != nil {
+ return fmt.Errorf("failed to get account user: %w", err)
+ }
+
+ _, err = deductFeeAllInCashBalance(ctx, tx, input, remainValue, acctUser.CashBalance)
+ if err != nil {
+ return fmt.Errorf("deduct all in balance, error:%w", err)
+ }
+ }
+
+ return nil
+}
+
+func deductFeeToBalance(ctx context.Context, tx bun.Tx, input AccountStatement, feeValue, balanceValue float64) (float64, error) {
+ if balanceValue <= 0 {
+ return 0, fmt.Errorf("balance is not enough")
+ }
+ statementValue := 0.0
+ remainValue := balanceValue + feeValue
+ if remainValue >= 0 {
+ remainValue = 0.0
+ statementValue = feeValue
+ } else {
+ statementValue = 0 - balanceValue
+ }
+
+ input.Value = statementValue
+ input.BalanceValue = balanceValue + statementValue
+ input.BalanceType = types.ChargeBalance
+
+ runSql := "update account_users set balance=balance + ? where user_uuid=?"
+ if err := assertAffectedOneRow(tx.Exec(runSql, input.Value, input.UserUUID)); err != nil {
+ return 0, fmt.Errorf("update balance, error:%w", err)
+ }
+
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&input).Exec(ctx)); err != nil {
+ return 0, fmt.Errorf("insert statement for deduct balance, error:%w", err)
+ }
+ return remainValue, nil
+}
+
+func deductFeeToCashBalance(ctx context.Context, tx bun.Tx, input AccountStatement, feeValue, cashBalanceValue float64) (float64, error) {
+ if cashBalanceValue <= 0 {
+ return 0, fmt.Errorf("cash balance is not enough")
+ }
+ statementValue := 0.0
+ remainValue := cashBalanceValue + feeValue
+ if remainValue >= 0 {
+ remainValue = 0.0
+ statementValue = feeValue
+ } else {
+ statementValue = 0 - cashBalanceValue
+ }
+
+ input.Value = statementValue
+ input.BalanceValue = cashBalanceValue + statementValue
+ input.BalanceType = types.ChargeCashBalance
+
+ runSql := "update account_users set cash_balance=cash_balance + ? where user_uuid=?"
+ if err := assertAffectedOneRow(tx.Exec(runSql, input.Value, input.UserUUID)); err != nil {
+ return 0, fmt.Errorf("update cash balance, error:%w", err)
+ }
+
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&input).Exec(ctx)); err != nil {
+ return 0, fmt.Errorf("insert statement for deduct cash balance, error:%w", err)
+ }
+ return remainValue, nil
+}
+
+func deductFeeAllInCashBalance(ctx context.Context, tx bun.Tx, input AccountStatement, feeValue, cashBalanceValue float64) (float64, error) {
+ remainValue := cashBalanceValue + feeValue
+
+ input.Value = feeValue
+ input.BalanceValue = remainValue
+ input.BalanceType = types.ChargeCashBalance
+
+ runSql := "update account_users set cash_balance=cash_balance + ? where user_uuid=?"
+ if err := assertAffectedOneRow(tx.Exec(runSql, input.Value, input.UserUUID)); err != nil {
+ return 0, fmt.Errorf("update balance for allin, error:%w", err)
+ }
+
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&input).Exec(ctx)); err != nil {
+ return 0, fmt.Errorf("insert statement for deduct allin balance, error:%w", err)
+ }
+
+ return remainValue, nil
+}
+
+func CheckDuplicatedEvent(ctx context.Context, tx bun.Tx, input AccountStatement) error {
+ var err error
+ var acctStatement AccountStatement
+
+ err = tx.NewSelect().Model(&acctStatement).Where("event_uuid = ?", input.EventUUID).Scan(ctx, &acctStatement)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("verfiy account statement, event_uuid: %s, err: %w", input.EventUUID, err)
+ }
+ } else {
+ if acctStatement.EventUUID == input.EventUUID {
+ return types.ErrDuplicatedEvent
+ }
+ }
+ return nil
+}
+
+func updateFeeBill(ctx context.Context, tx bun.Tx, input AccountStatement) error {
+ if !utils.IsNeedCalculateBill(input.Scene) {
+ return nil
+ }
+ // calculate bill
+ bill := AccountBill{
+ BillDate: input.EventDate,
+ UserUUID: input.UserUUID,
+ Scene: input.Scene,
+ CustomerID: input.CustomerID,
+ Value: input.Value,
+ Consumption: input.Consumption,
+ }
+ err := tx.NewSelect().Model(&bill).Where("bill_date = ? and user_uuid = ? and scene = ? and customer_id = ?", input.EventDate, input.UserUUID, input.Scene, input.CustomerID).For("UPDATE").Scan(ctx)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("select statement, error:%w", err)
+ }
+ if errors.Is(err, sql.ErrNoRows) {
+ _, err = tx.NewInsert().Model(&bill).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("create statement, error:%w", err)
+ }
+ } else {
+ _, err = tx.NewUpdate().Model(&bill).Where("bill_date = ? and user_uuid = ? and scene = ? and customer_id = ?", input.EventDate, input.UserUUID, input.Scene, input.CustomerID).Set("value = value + ?, consumption = consumption + ?, updated_at=current_timestamp", input.Value, input.Consumption).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("update statement, error:%w", err)
+ }
+ }
+
+ return nil
+}
+
+func (as *accountStatementStoreImpl) ListByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (AccountStatementRes, error) {
+ var accountStatment []AccountStatement
+ q := as.db.Operator.Core.NewSelect().Model(&accountStatment).Where("user_uuid = ? and scene = ? and customer_id = ? and created_at >= ? and created_at <= ?", req.UserUUID, req.Scene, req.InstanceName, req.StartTime, req.EndTime)
+
+ count, err := q.Count(ctx)
+ if err != nil {
+ return AccountStatementRes{}, fmt.Errorf("count statement, error:%w", err)
+ }
+
+ var totalResult TotalResult
+ err = as.db.Operator.Core.NewSelect().With("grouped_items", q).TableExpr("grouped_items").ColumnExpr("SUM(value) AS total_value, SUM(consumption) as total_consumption").Scan(ctx, &totalResult)
+ if err != nil {
+ return AccountStatementRes{}, fmt.Errorf("group statement, error:%w", err)
+ }
+
+ _, err = q.Order("id DESC").Limit(req.Per).Offset((req.Page-1)*req.Per).Exec(ctx, &accountStatment)
+ if err != nil {
+ return AccountStatementRes{}, fmt.Errorf("list statement, error:%w", err)
+ }
+ return AccountStatementRes{
+ Data: accountStatment,
+ ACCT_SUMMARY: types.ACCT_SUMMARY{
+ Total: count,
+ TotalValue: totalResult.TotalValue,
+ TotalConsumption: totalResult.TotalConsumption},
+ }, nil
+}
+
+func (as *accountStatementStoreImpl) GetByEventID(ctx context.Context, eventID uuid.UUID) (AccountStatement, error) {
+ var result AccountStatement
+ _, err := as.db.Core.NewSelect().Model(&result).Where("event_uuid = ?", eventID).Exec(ctx, &result)
+ if err != nil {
+ return result, fmt.Errorf("get statement, error:%w", err)
+ }
+ return result, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type accountSyncQuotaStoreImpl struct {
+ db *DB
+}
+
+type AccountSyncQuotaStore interface {
+ GetByID(ctx context.Context, userID int64) (*AccountSyncQuota, error)
+ Update(ctx context.Context, accountQuota AccountSyncQuota) (*AccountSyncQuota, error)
+ Create(ctx context.Context, accountQuota AccountSyncQuota) error
+ Delete(ctx context.Context, accountQuota AccountSyncQuota) error
+ ListAllByUserID(ctx context.Context, userID int64) ([]AccountSyncQuota, error)
+}
+
+func NewAccountSyncQuotaStore() AccountSyncQuotaStore {
+ return &accountSyncQuotaStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountSyncQuotaStoreWithDB(db *DB) AccountSyncQuotaStore {
+ return &accountSyncQuotaStoreImpl{
+ db: db,
+ }
+}
+
+type AccountSyncQuota struct {
+ UserID int64 `bun:",pk" json:"user_id"`
+ RepoCountLimit int64 `bun:",notnull" json:"repo_count_limit"`
+ RepoCountUsed int64 `bun:",notnull" json:"repo_count_used"`
+ SpeedLimit int64 `bun:",notnull" json:"speed_limit"`
+ TrafficLimit int64 `bun:",notnull" json:"traffic_limit"`
+ TrafficUsed int64 `bun:",notnull" json:"traffic_used"`
+}
+
+func (s *accountSyncQuotaStoreImpl) GetByID(ctx context.Context, userID int64) (*AccountSyncQuota, error) {
+ quota := &AccountSyncQuota{}
+ err := s.db.Core.NewSelect().Model(quota).Where("user_id = ?", userID).Scan(ctx, quota)
+ return quota, err
+}
+
+func (s *accountSyncQuotaStoreImpl) Update(ctx context.Context, accountQuota AccountSyncQuota) (*AccountSyncQuota, error) {
+ res, err := s.db.Core.NewUpdate().Model(&accountQuota).WherePK().Column("repo_count_limit", "speed_limit", "traffic_limit").Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("update user quota failed, error:%w", err)
+ }
+ return &accountQuota, err
+}
+
+func (s *accountSyncQuotaStoreImpl) Create(ctx context.Context, accountQuota AccountSyncQuota) error {
+ res, err := s.db.Core.NewInsert().Model(&accountQuota).Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("insert user quota failed, error:%w", err)
+ }
+ return nil
+}
+
+func (s *accountSyncQuotaStoreImpl) Delete(ctx context.Context, accountQuota AccountSyncQuota) error {
+ _, err := s.db.Core.NewDelete().Model(&accountQuota).WherePK().Exec(ctx)
+ return err
+}
+
+func (am *accountSyncQuotaStoreImpl) ListAllByUserID(ctx context.Context, userID int64) ([]AccountSyncQuota, error) {
+ var accountSyncQuotas []AccountSyncQuota
+ err := am.db.Operator.Core.NewSelect().Model(&accountSyncQuotas).Where("user_id = ?", userID).Scan(ctx, &accountSyncQuotas)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list all account sync quotas by user id: %w", err)
+ }
+ return accountSyncQuotas, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountSyncQuotaStatementStoreImpl struct {
+ db *DB
+}
+
+type AccountSyncQuotaStatementStore interface {
+ Create(ctx context.Context, acctQuotaStatement AccountSyncQuotaStatement) error
+ Get(ctx context.Context, userID int64, req types.ACCT_QUOTA_STATEMENT_REQ) (*AccountSyncQuotaStatement, error)
+}
+
+func NewAccountSyncQuotaStatementStore() AccountSyncQuotaStatementStore {
+ return &accountSyncQuotaStatementStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountSyncQuotaStatementStoreWithDB(db *DB) AccountSyncQuotaStatementStore {
+ return &accountSyncQuotaStatementStoreImpl{
+ db: db,
+ }
+}
+
+type AccountSyncQuotaStatement struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ RepoPath string `bun:",notnull" json:"repo_path"`
+ RepoType string `bun:",notnull" json:"repo_type"`
+ CreatedAt time.Time `bun:",notnull,default:current_timestamp" json:"created_at"`
+}
+
+func (s *accountSyncQuotaStatementStoreImpl) Create(ctx context.Context, acctQuotaStatement AccountSyncQuotaStatement) error {
+ err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&acctQuotaStatement).Exec(ctx)); err != nil {
+ return err
+ }
+
+ runSql := "update account_sync_quota set repo_count_used=repo_count_used+1 where user_id = ? and repo_count_limit>repo_count_used"
+ if err := assertAffectedOneRow(tx.Exec(runSql, acctQuotaStatement.UserID)); err != nil {
+ return errors.New("repo download reach limit")
+ }
+
+ return nil
+ })
+
+ return err
+}
+
+func (s *accountSyncQuotaStatementStoreImpl) Get(ctx context.Context, userID int64, req types.ACCT_QUOTA_STATEMENT_REQ) (*AccountSyncQuotaStatement, error) {
+ quotaStatement := &AccountSyncQuotaStatement{}
+ err := s.db.Core.NewSelect().Model(quotaStatement).Where("user_id = ? and repo_path = ? and repo_type = ?", userID, req.RepoPath, req.RepoType).Scan(ctx, quotaStatement)
+ return quotaStatement, err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type accountUserStoreImpl struct {
+ db *DB
+}
+
+type AccountUserStore interface {
+ List(ctx context.Context, per, page int) ([]AccountUser, int, error)
+ Create(ctx context.Context, input AccountUser) error
+ FindUserByID(ctx context.Context, userID string) (*AccountUser, error)
+ ListAllByUserUUID(ctx context.Context, userUUID string) ([]AccountUser, error)
+}
+
+func NewAccountUserStore() AccountUserStore {
+ return &accountUserStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewAccountUserStoreWithDB(db *DB) AccountUserStore {
+ return &accountUserStoreImpl{
+ db: db,
+ }
+}
+
+type AccountUser struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ Balance float64 `bun:",notnull" json:"balance"`
+ CashBalance float64 `bun:",notnull" json:"cash_balance"`
+}
+
+func (s *accountUserStoreImpl) List(ctx context.Context, per, page int) ([]AccountUser, int, error) {
+ var result []AccountUser
+ q := s.db.Operator.Core.NewSelect().Model(&result)
+ count, err := q.Count(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ _, err = q.Order("user_uuid").Limit(per).Offset((page-1)*per).Exec(ctx, &result)
+ if err != nil {
+ return nil, 0, fmt.Errorf("list all accounts, error:%w", err)
+ }
+ return result, count, nil
+}
+
+func (s *accountUserStoreImpl) Create(ctx context.Context, input AccountUser) error {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("create user balance failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *accountUserStoreImpl) FindUserByID(ctx context.Context, userID string) (*AccountUser, error) {
+ user := &AccountUser{}
+ err := s.db.Core.NewSelect().Model(user).Where("user_uuid = ?", userID).Scan(ctx, user)
+ return user, err
+}
+
+func (am *accountUserStoreImpl) ListAllByUserUUID(ctx context.Context, userUUID string) ([]AccountUser, error) {
+ var accountUsers []AccountUser
+ err := am.db.Operator.Core.NewSelect().Model(&accountUsers).Where("user_uuid = ?", userUUID).Scan(ctx, &accountUsers)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list all account users by user uuid: %w", err)
+ }
+ return accountUsers, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type argoWorkFlowStoreImpl struct {
+ db *DB
+}
+
+type ArgoWorkFlowStore interface {
+ FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error)
+ FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error)
+ FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error)
+ CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error)
+ // mainly for update status
+ UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error)
+ // delete workflow by id
+ DeleteWorkFlow(ctx context.Context, id int64) error
+}
+
+func NewArgoWorkFlowStore() ArgoWorkFlowStore {
+ return &argoWorkFlowStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewArgoWorkFlowStoreWithDB(db *DB) ArgoWorkFlowStore {
+ return &argoWorkFlowStoreImpl{
+ db: db,
+ }
+}
+
+type ArgoWorkflow struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Username string `bun:",notnull" json:"username"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ TaskName string `bun:",notnull" json:"task_name"` // user input name
+ TaskId string `bun:",notnull" json:"task_id"` // generated task id
+ TaskType types.TaskType `bun:",notnull" json:"task_type"`
+ ClusterID string `bun:",notnull" json:"cluster_id"`
+ Namespace string `bun:",notnull" json:"namespace"`
+ RepoIds []string `bun:",notnull,type:jsonb" json:"repo_ids"`
+ RepoType string `bun:",notnull" json:"repo_type"`
+ TaskDesc string `bun:"," json:"task_desc"`
+ Status v1alpha1.WorkflowPhase `bun:"," json:"status"`
+ Reason string `bun:"," json:"reason"` // reason for status
+ Image string `bun:",notnull" json:"image"` // ArgoWorkFlow framework
+ Datasets []string `bun:",notnull,type:jsonb" json:"datasets"`
+ ResourceId int64 `bun:",nullzero" json:"resource_id"`
+ ResourceName string `bun:"," json:"resource_name"`
+ SubmitTime time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"submit_time"`
+ StartTime time.Time `bun:",nullzero" json:"start_time"`
+ EndTime time.Time `bun:",nullzero" json:"end_time"`
+ ResultURL string `bun:"," json:"result_url"`
+ DownloadURL string `bun:"," json:"download_url"`
+ FailuresURL string `bun:"," json:"failures_url"`
+}
+
+func (s *argoWorkFlowStoreImpl) FindByID(ctx context.Context, id int64) (WorkFlow ArgoWorkflow, err error) {
+ err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("id = ?", id).Scan(ctx, &WorkFlow)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *argoWorkFlowStoreImpl) FindByTaskID(ctx context.Context, id string) (WorkFlow ArgoWorkflow, err error) {
+ err = s.db.Operator.Core.NewSelect().Model(&WorkFlow).Where("task_id = ?", id).Scan(ctx, &WorkFlow)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *argoWorkFlowStoreImpl) FindByUsername(ctx context.Context, username string, per, page int) (WorkFlows []ArgoWorkflow, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&WorkFlows).
+ Where("username = ?", username)
+
+ query = query.Order("submit_time DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *argoWorkFlowStoreImpl) CreateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) {
+ res, err := s.db.Core.NewInsert().Model(&workFlow).Exec(ctx, &workFlow)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("failed to save WorkFlow in db, error:%w", err)
+ }
+
+ return &workFlow, nil
+}
+
+// mainly for update status
+func (s *argoWorkFlowStoreImpl) UpdateWorkFlow(ctx context.Context, workFlow ArgoWorkflow) (*ArgoWorkflow, error) {
+ _, err := s.db.Core.NewUpdate().Model(&workFlow).WherePK().Exec(ctx)
+ return &workFlow, err
+}
+
+// delete workflow by id
+func (s *argoWorkFlowStoreImpl) DeleteWorkFlow(ctx context.Context, id int64) error {
+ _, err := s.db.Core.NewDelete().Model(&ArgoWorkflow{}).Where("id = ?", id).Exec(ctx)
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+)
+
+type clusterInfoStoreImpl struct {
+ db *DB
+}
+
+type ClusterInfoStore interface {
+ Add(ctx context.Context, clusterConfig string, region string) error
+ Update(ctx context.Context, clusterInfo ClusterInfo) error
+ ByClusterID(ctx context.Context, clusterId string) (clusterInfo ClusterInfo, err error)
+ ByClusterConfig(ctx context.Context, clusterConfig string) (clusterInfo ClusterInfo, err error)
+ List(ctx context.Context) ([]ClusterInfo, error)
+}
+
+func NewClusterInfoStore() ClusterInfoStore {
+ return &clusterInfoStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewClusterInfoStoreWithDB(db *DB) ClusterInfoStore {
+ return &clusterInfoStoreImpl{
+ db: db,
+ }
+}
+
+type ClusterInfo struct {
+ ClusterID string `bun:",pk" json:"cluster_id"`
+ ClusterConfig string `bun:",notnull" json:"cluster_config"`
+ StorageClass string `bun:",notnull" json:"storage_class"`
+ Region string `bun:",notnull" json:"region"`
+ Zone string `bun:",notnull" json:"zone"` //cn-beijing
+ Provider string `bun:",notnull" json:"provider"` //ali
+ Enable bool `bun:",notnull" json:"enable"`
+}
+
+func (r *clusterInfoStoreImpl) Add(ctx context.Context, clusterConfig string, region string) error {
+ err := r.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ cluster := &ClusterInfo{
+ ClusterID: uuid.New().String(),
+ ClusterConfig: clusterConfig,
+ Region: region,
+ Enable: true,
+ }
+
+ _, err := r.ByClusterConfig(ctx, clusterConfig)
+ if errors.Is(err, sql.ErrNoRows) {
+ return assertAffectedOneRow(r.db.Operator.Core.NewInsert().Model(cluster).Exec(ctx))
+ }
+ return err
+ })
+ return err
+}
+
+func (r *clusterInfoStoreImpl) Update(ctx context.Context, clusterInfo ClusterInfo) error {
+ err := r.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ _, err := r.ByClusterConfig(ctx, clusterInfo.ClusterConfig)
+ if err == nil {
+ return assertAffectedOneRow(r.db.Operator.Core.NewUpdate().Model(&clusterInfo).WherePK().Exec(ctx))
+ }
+ return nil
+ })
+ return err
+}
+
+func (s *clusterInfoStoreImpl) ByClusterID(ctx context.Context, clusterId string) (clusterInfo ClusterInfo, err error) {
+ clusterInfo.ClusterID = clusterId
+ err = s.db.Operator.Core.NewSelect().Model(&clusterInfo).Where("cluster_id = ?", clusterId).Scan(ctx)
+ return
+}
+
+func (s *clusterInfoStoreImpl) ByClusterConfig(ctx context.Context, clusterConfig string) (clusterInfo ClusterInfo, err error) {
+ clusterInfo.ClusterConfig = clusterConfig
+ err = s.db.Operator.Core.NewSelect().Model(&clusterInfo).Where("cluster_config = ?", clusterConfig).Scan(ctx)
+ return
+}
+
+func (s *clusterInfoStoreImpl) List(ctx context.Context) ([]ClusterInfo, error) {
+ var result []ClusterInfo
+ _, err := s.db.Operator.Core.NewSelect().Model(&result).Order("region").Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+type codeStoreImpl struct {
+ db *DB
+}
+
+type CodeStore interface {
+ ByRepoIDs(ctx context.Context, repoIDs []int64) (codes []Code, err error)
+ ByRepoID(ctx context.Context, repoID int64) (*Code, error)
+ ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (codes []Code, total int, err error)
+ UserLikesCodes(ctx context.Context, userID int64, per, page int) (codes []Code, total int, err error)
+ ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (codes []Code, total int, err error)
+ Create(ctx context.Context, input Code) (*Code, error)
+ Update(ctx context.Context, input Code) (err error)
+ FindByPath(ctx context.Context, namespace string, repoPath string) (code *Code, err error)
+ Delete(ctx context.Context, input Code) error
+ ListByPath(ctx context.Context, paths []string) ([]Code, error)
+}
+
+func NewCodeStore() CodeStore {
+ return &codeStoreImpl{db: defaultDB}
+}
+
+func NewCodeStoreWithDB(db *DB) CodeStore {
+ return &codeStoreImpl{db: db}
+}
+
+type Code struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ LastUpdatedAt time.Time `bun:",notnull" json:"last_updated_at"`
+ times
+}
+
+func (s *codeStoreImpl) ByRepoIDs(ctx context.Context, repoIDs []int64) (codes []Code, err error) {
+ err = s.db.Operator.Core.NewSelect().
+ Model(&codes).
+ Where("repository_id in (?)", bun.In(repoIDs)).
+ Scan(ctx)
+
+ return
+}
+
+func (s *codeStoreImpl) ByRepoID(ctx context.Context, repoID int64) (*Code, error) {
+ var code Code
+ err := s.db.Operator.Core.NewSelect().
+ Model(&code).
+ Where("repository_id = ?", repoID).
+ Scan(ctx)
+
+ if err != nil {
+ return nil, fmt.Errorf("failed to select code, error: %w", err)
+ }
+
+ return &code, nil
+}
+
+func (s *codeStoreImpl) ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (codes []Code, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&codes).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", username))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("code.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *codeStoreImpl) UserLikesCodes(ctx context.Context, userID int64, per, page int) (codes []Code, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&codes).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.id in (select repo_id from user_likes where user_id=?)", userID)
+
+ query = query.Order("code.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *codeStoreImpl) ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (codes []Code, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&codes).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", namespace))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("code.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx, &codes)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *codeStoreImpl) Create(ctx context.Context, input Code) (*Code, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create code in db failed", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("create code in db failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *codeStoreImpl) Update(ctx context.Context, input Code) (err error) {
+ _, err = s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+ return
+}
+
+func (s *codeStoreImpl) FindByPath(ctx context.Context, namespace string, repoPath string) (code *Code, err error) {
+ resCode := new(Code)
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(resCode).
+ Relation("Repository.User").
+ Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, repoPath)).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find code: %w", err)
+ }
+ err = s.db.Operator.Core.NewSelect().
+ Model(resCode.Repository).
+ WherePK().
+ Relation("Tags", func(sq *bun.SelectQuery) *bun.SelectQuery {
+ return sq.Where("repository_tag.count > 0")
+ }).
+ Scan(ctx)
+ return resCode, err
+}
+
+func (s *codeStoreImpl) Delete(ctx context.Context, input Code) error {
+ res, err := s.db.Operator.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("delete code in tx failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *codeStoreImpl) ListByPath(ctx context.Context, paths []string) ([]Code, error) {
+ var codes []Code
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&Code{}).
+ Relation("Repository").
+ Where("path IN (?)", bun.In(paths)).
+ Scan(ctx, &codes)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find models by path,error: %w", err)
+ }
+ return codes, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "slices"
+ "strings"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type collectionStoreImpl struct {
+ db *DB
+}
+
+type CollectionStore interface {
+ // query collections in the database
+ GetCollections(ctx context.Context, filter *types.CollectionFilter, per, page int, showPrivate bool) (collections []Collection, total int, err error)
+ // query collections in the database
+ QueryByTrending(ctx context.Context, filter *types.CollectionFilter, per, page int) (collections []Collection, total int, err error)
+ CreateCollection(ctx context.Context, collection Collection) (*Collection, error)
+ DeleteCollection(ctx context.Context, id int64, uid int64) error
+ UpdateCollection(ctx context.Context, collection Collection) (*Collection, error)
+ GetCollection(ctx context.Context, id int64) (*Collection, error)
+ ByUserLikes(ctx context.Context, userID int64, per, page int) (collections []Collection, total int, err error)
+ ByUserOrgs(ctx context.Context, namespace string, per, page int, onlyPublic bool) (collections []Collection, total int, err error)
+ // get collections by ids
+ GetCollectionsByIDs(ctx context.Context, collections []Collection, ids []interface{}, total int, onlyPublic bool) ([]Collection, int, error)
+ FindById(ctx context.Context, id int64) (collection Collection, err error)
+ AddCollectionRepos(ctx context.Context, crs []CollectionRepository) error
+ RemoveCollectionRepos(ctx context.Context, crs []CollectionRepository) error
+ ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (collections []Collection, total int, err error)
+}
+
+func NewCollectionStore() CollectionStore {
+ return &collectionStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewCollectionStoreWithDB(db *DB) CollectionStore {
+ return &collectionStoreImpl{
+ db: db,
+ }
+}
+
+type Collection struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Namespace string `bun:",notnull" json:"namespace"`
+ Username string `bun:",notnull" json:"username"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ Name string `bun:",notnull" json:"name"`
+ Theme string `bun:",notnull" json:"theme"`
+ Nickname string `bun:",notnull" json:"nickname"`
+ Description string `bun:",nullzero" json:"description"`
+ Private bool `bun:",notnull" json:"private"`
+ Repositories []Repository `bun:"m2m:collection_repositories,join:Collection=Repository" json:"repositories"`
+ Likes int64 `bun:",nullzero" json:"likes"`
+ times
+}
+
+type CollectionRepository struct {
+ ID int64 `bun:",autoincrement" json:"id"`
+ CollectionID int64 `bun:",pk" json:"collection_id"`
+ RepositoryID int64 `bun:",pk" json:"repository_id"`
+ Collection *Collection `bun:"rel:belongs-to,join:collection_id=id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id"`
+}
+
+type RankedRepository struct {
+ CollectionID int64 `bun:"collection_id"`
+ RepositoryID int64 `bun:"repository_id"`
+ RN int `bun:"rn"` // Rank
+}
+
+var Fields = []string{"id", "download_count", "likes", "path", "private", "repository_type", "updated_at", "created_at", "user_id", "name", "nickname", "description"}
+
+// query collections in the database
+func (cs *collectionStoreImpl) GetCollections(ctx context.Context, filter *types.CollectionFilter, per, page int, showPrivate bool) (collections []Collection, total int, err error) {
+ if filter.Sort == "trending" {
+ return cs.QueryByTrending(ctx, filter, per, page)
+ }
+ query := cs.db.Operator.Core.
+ NewSelect().
+ Model(&collections).
+ Where("private = ?", false)
+ if filter.Search != "" {
+ filter.Search = strings.ToLower(filter.Search)
+ query.Where(
+ "LOWER(name) like ?", fmt.Sprintf("%%%s%%", filter.Search),
+ )
+ }
+ err = query.Order(sortBy[filter.Sort]).
+ Limit(per).Offset((page - 1) * per).
+ Scan(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+
+ ids := make([]interface{}, 0)
+ for _, collection := range collections {
+ ids = append(ids, collection.ID)
+ }
+ return cs.GetCollectionsByIDs(ctx, collections, ids, total, true)
+}
+
+// query collections in the database
+func (cs *collectionStoreImpl) QueryByTrending(ctx context.Context, filter *types.CollectionFilter, per, page int) (collections []Collection, total int, err error) {
+ query := cs.db.Operator.Core.NewSelect().
+ Model(&collections).
+ Column("collection.*").
+ ColumnExpr("SUM(COALESCE(rors.score, 0)+COALESCE(ropw.weight, 0)) AS popularity").
+ Join("LEFT JOIN collection_repositories cr ON collection.id = cr.collection_id ").
+ Join("LEFT JOIN repositories r ON cr.repository_id = r.id").
+ Join("LEFT JOIN recom_op_weights ropw ON r.id = ropw.repository_id").
+ Join("LEFT JOIN recom_repo_scores rors ON r.id = rors.repository_id")
+ query.Where("collection.private = ?", false)
+ if filter.Search != "" {
+ filter.Search = strings.ToLower(filter.Search)
+ query.Where(
+ "LOWER(collection.name) like ?", fmt.Sprintf("%%%s%%", filter.Search),
+ )
+ }
+ query.Group("collection.id")
+ err = query.Order(sortBy[filter.Sort]).
+ Limit(per).Offset((page - 1) * per).
+ Scan(ctx)
+ if err != nil || len(collections) == 0 {
+ return nil, 0, err
+ }
+
+ total, err = query.Count(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ ids := make([]interface{}, 0)
+ for _, collection := range collections {
+ ids = append(ids, collection.ID)
+ }
+
+ return cs.GetCollectionsByIDs(ctx, collections, ids, total, true)
+}
+
+func (cs *collectionStoreImpl) CreateCollection(ctx context.Context, collection Collection) (*Collection, error) {
+ res, err := cs.db.Core.NewInsert().Model(&collection).Exec(ctx, &collection)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("failed to create collection in db, error:%w", err)
+ }
+
+ return &collection, nil
+}
+
+func (cs *collectionStoreImpl) DeleteCollection(ctx context.Context, id int64, uid int64) error {
+ var collection Collection
+ res, err := cs.db.Operator.Core.NewDelete().Model(&collection).Where("id =?", id).Where("user_id =?", uid).Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("failed to delete collection in db, error:%w", err)
+ }
+ return nil
+}
+
+func (cs *collectionStoreImpl) UpdateCollection(ctx context.Context, collection Collection) (*Collection, error) {
+
+ _, err := cs.db.Core.NewUpdate().Model(&collection).WherePK().Exec(ctx)
+ return &collection, err
+}
+
+func (cs *collectionStoreImpl) GetCollection(ctx context.Context, id int64) (*Collection, error) {
+ collection := new(Collection)
+ err := cs.db.Operator.Core.
+ NewSelect().
+ Model(collection).
+ Relation("Repositories.Tags", func(q *bun.SelectQuery) *bun.SelectQuery {
+ return q.Where("category = ?", "task")
+ }).
+ Relation("Repositories", func(q *bun.SelectQuery) *bun.SelectQuery {
+ return q.Column(Fields...).OrderExpr("updated_at DESC")
+ }).
+ Where("id =?", id).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("can not find collection: %w", err)
+ }
+
+ return collection, err
+}
+
+func (cs *collectionStoreImpl) ByUserLikes(ctx context.Context, userID int64, per, page int) (collections []Collection, total int, err error) {
+ query := cs.db.Operator.Core.
+ NewSelect().
+ Model(&collections).
+ Where("collection.id in (select collection_id from user_likes where user_id=?)", userID)
+
+ query = query.Order("collection.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ ids := make([]interface{}, 0)
+ for _, collection := range collections {
+ ids = append(ids, collection.ID)
+ }
+
+ return cs.GetCollectionsByIDs(ctx, collections, ids, total, true)
+}
+
+func (cs *collectionStoreImpl) ByUserOrgs(ctx context.Context, namespace string, per, page int, onlyPublic bool) (collections []Collection, total int, err error) {
+ query := cs.db.Operator.Core.
+ NewSelect().
+ Model(&collections).
+ Where("collection.namespace = ?", namespace)
+
+ if onlyPublic {
+ query = query.Where("collection.private = ?", false)
+ }
+
+ query = query.Order("collection.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ ids := make([]interface{}, 0)
+ for _, collection := range collections {
+ ids = append(ids, collection.ID)
+ }
+
+ return cs.GetCollectionsByIDs(ctx, collections, ids, total, false)
+}
+
+// get collections by ids
+func (cs *collectionStoreImpl) GetCollectionsByIDs(ctx context.Context, collections []Collection, ids []interface{}, total int, onlyPublic bool) ([]Collection, int, error) {
+ subQuery := cs.db.Operator.Core.NewSelect().
+ Column("cr.collection_id").
+ ColumnExpr("repository.id as repository_id").
+ ColumnExpr("ROW_NUMBER() OVER (PARTITION BY cr.collection_id ORDER BY repository.updated_at DESC) AS rn").
+ TableExpr("repositories AS repository")
+ if onlyPublic {
+ subQuery.Where("repository.private = ?", false)
+ }
+ subQuery.Join("JOIN collection_repositories AS cr ON repository.id = cr.repository_id")
+
+ var rankedRepos []RankedRepository
+ err := cs.db.Operator.Core.NewSelect().
+ With("rn_repositories", subQuery).
+ TableExpr("rn_repositories").
+ Where("rn <= ?", 3).
+ Where("collection_id IN (?)", bun.In(ids)).
+ Scan(ctx, &rankedRepos)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ repo_ids := make([]int64, 0)
+ for _, rr := range rankedRepos {
+ if !slices.Contains(repo_ids, rr.RepositoryID) {
+ repo_ids = append(repo_ids, rr.RepositoryID)
+ }
+ }
+ var repositories []Repository
+ err = cs.db.Operator.Core.NewSelect().
+ Model(&repositories).
+ Column(Fields...).
+ Relation("Tags", func(q *bun.SelectQuery) *bun.SelectQuery {
+ return q.Where("category = ?", "task")
+ }).
+ Where("repository.id IN (?)", bun.In(repo_ids)).
+ Order("updated_at DESC").
+ Scan(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ collectionMaps := getCollectionMaps(rankedRepos, repositories)
+ for i, collection := range collections {
+ collections[i].Repositories = collectionMaps[collection.ID]
+ }
+
+ return collections, total, nil
+}
+
+// return collection maps from rankedRepos and repositories
+func getCollectionMaps(rankedRepos []RankedRepository, repositories []Repository) (collections map[int64][]Repository) {
+ collections = make(map[int64][]Repository)
+ repoMap := make(map[int64]Repository)
+ for _, repo := range repositories {
+ repoMap[repo.ID] = repo
+ }
+ for _, rr := range rankedRepos {
+ collections[rr.CollectionID] = append(collections[rr.CollectionID], repoMap[rr.RepositoryID])
+ }
+ return
+}
+
+func (cs *collectionStoreImpl) FindById(ctx context.Context, id int64) (collection Collection, err error) {
+ q := cs.db.Operator.Core.
+ NewSelect()
+ err = q.
+ Model(&collection).
+ Where("id =?", id).
+ Scan(ctx)
+ return
+}
+
+func (cs *collectionStoreImpl) AddCollectionRepos(ctx context.Context, crs []CollectionRepository) error {
+
+ result, err := cs.db.Core.NewInsert().Model(&crs).Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return assertAffectedXRows(int64(len(crs)), result, err)
+}
+
+func (cs *collectionStoreImpl) RemoveCollectionRepos(ctx context.Context, crs []CollectionRepository) error {
+ for _, cr := range crs {
+ _, err := cs.db.Core.NewDelete().
+ Model((*CollectionRepository)(nil)).
+ Where("collection_id = ? AND repository_id = ?", cr.CollectionID, cr.RepositoryID).
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to remove repo %d from collection %d, error: %w", cr.RepositoryID, cr.CollectionID, err)
+ }
+ }
+ return nil
+}
+
+func (cs *collectionStoreImpl) ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (collections []Collection, total int, err error) {
+ query := cs.db.Operator.Core.
+ NewSelect().
+ Model(&collections).
+ Relation("Repositories.Tags", func(q *bun.SelectQuery) *bun.SelectQuery {
+ return q.Where("category = ?", "task")
+ }).
+ Relation("Repositories", func(q *bun.SelectQuery) *bun.SelectQuery {
+ return q.Column(Fields...).OrderExpr("updated_at DESC").Limit(3)
+ }).
+ Where("collection.username = ?", username)
+
+ if onlyPublic {
+ query = query.Where("collection.private = ?", false)
+ }
+ query = query.Order("collection.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "time"
+
+ "github.com/uptrace/bun"
+ "github.com/uptrace/bun/schema"
+)
+
+type times struct {
+ CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"updated_at"`
+}
+
+func (t *times) BeforeAppendModel(ctx context.Context, query schema.Query) error {
+ switch query.(type) {
+ case *bun.UpdateQuery:
+ q := query.(*bun.UpdateQuery)
+ m := q.GetModel().Value()
+ //skip update for Repository, and related assets
+ if _, ok := m.(*Repository); ok {
+ return nil
+ }
+ if _, ok := m.(*Model); ok {
+ return nil
+ }
+ if _, ok := m.(*Dataset); ok {
+ return nil
+ }
+ if _, ok := m.(*Code); ok {
+ return nil
+ }
+ if _, ok := m.(*Space); ok {
+ return nil
+ }
+ t.UpdatedAt = time.Now()
+ }
+
+ return nil
+}
+
+func assertAffectedOneRow(result sql.Result, err error) error {
+ return assertAffectedXRows(1, result, err)
+}
+
+func assertAffectedXRows(X int64, result sql.Result, err error) error {
+ if err != nil {
+ return err
+ }
+
+ affected, err := result.RowsAffected()
+ if err != nil {
+ err = fmt.Errorf("retrieving affected row count: %w", err)
+ return err
+ }
+ if affected != X {
+ err = fmt.Errorf("affected %d row(s), want %d", affected, X)
+ return err
+ }
+
+ return nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+var sortBy = map[string]string{
+ "trending": "popularity DESC NULLS LAST",
+ "recently_update": "updated_at DESC NULLS LAST",
+ "most_download": "download_count DESC NULLS LAST",
+ "most_favorite": "likes DESC NULLS LAST",
+}
+
+type datasetStoreImpl struct {
+ db *DB
+}
+
+type DatasetStore interface {
+ ByRepoIDs(ctx context.Context, repoIDs []int64) (datasets []Dataset, err error)
+ ByRepoID(ctx context.Context, repoID int64) (*Dataset, error)
+ ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (datasets []Dataset, total int, err error)
+ UserLikesDatasets(ctx context.Context, userID int64, per, page int) (datasets []Dataset, total int, err error)
+ ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (datasets []Dataset, total int, err error)
+ Create(ctx context.Context, input Dataset) (*Dataset, error)
+ Update(ctx context.Context, input Dataset) (err error)
+ FindByPath(ctx context.Context, namespace string, repoPath string) (dataset *Dataset, err error)
+ Delete(ctx context.Context, input Dataset) error
+ ListByPath(ctx context.Context, paths []string) ([]Dataset, error)
+ CreateIfNotExist(ctx context.Context, input Dataset) (*Dataset, error)
+}
+
+func NewDatasetStore() DatasetStore {
+ return &datasetStoreImpl{db: defaultDB}
+}
+
+func NewDatasetStoreWithDB(db *DB) DatasetStore {
+ return &datasetStoreImpl{db: db}
+}
+
+type Dataset struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ LastUpdatedAt time.Time `bun:",notnull" json:"last_updated_at"`
+ times
+}
+
+func (s *datasetStoreImpl) ByRepoIDs(ctx context.Context, repoIDs []int64) (datasets []Dataset, err error) {
+ q := s.db.Operator.Core.NewSelect().
+ Model(&datasets).
+ Relation("Repository").
+ Relation("Repository.User").
+ Where("repository_id in (?)", bun.In(repoIDs))
+ err = q.Scan(ctx)
+ return
+}
+
+func (s *datasetStoreImpl) ByRepoID(ctx context.Context, repoID int64) (*Dataset, error) {
+ var dataset Dataset
+ err := s.db.Operator.Core.NewSelect().
+ Model(&dataset).
+ Where("repository_id = ?", repoID).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to select dataset by repository id: %d, error: %w", repoID, err)
+ }
+
+ return &dataset, nil
+}
+
+func (s *datasetStoreImpl) ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (datasets []Dataset, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&datasets).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", username))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("dataset.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *datasetStoreImpl) UserLikesDatasets(ctx context.Context, userID int64, per, page int) (datasets []Dataset, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&datasets).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.id in (select repo_id from user_likes where user_id=?)", userID)
+
+ query = query.Order("dataset.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *datasetStoreImpl) ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (datasets []Dataset, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&datasets).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", namespace))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("dataset.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx, &datasets)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *datasetStoreImpl) Create(ctx context.Context, input Dataset) (*Dataset, error) {
+ input.LastUpdatedAt = time.Now()
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create dataset in db failed", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("create dataset in db failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *datasetStoreImpl) Update(ctx context.Context, input Dataset) (err error) {
+ input.LastUpdatedAt = time.Now()
+ _, err = s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+ return
+}
+
+func (s *datasetStoreImpl) FindByPath(ctx context.Context, namespace string, repoPath string) (dataset *Dataset, err error) {
+ resDataset := new(Dataset)
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(resDataset).
+ Relation("Repository.User").
+ Relation("Repository.Mirror").
+ Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, repoPath)).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset: %w", err)
+ }
+ err = s.db.Operator.Core.NewSelect().
+ Model(resDataset.Repository).
+ WherePK().
+ Relation("Tags", func(sq *bun.SelectQuery) *bun.SelectQuery {
+ return sq.Where("repository_tag.count > 0")
+ }).
+ Scan(ctx)
+ return resDataset, err
+}
+
+func (s *datasetStoreImpl) Delete(ctx context.Context, input Dataset) error {
+ res, err := s.db.Operator.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("delete dataset in tx failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *datasetStoreImpl) ListByPath(ctx context.Context, paths []string) (datasets []Dataset, err error) {
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(&datasets).
+ Relation("Repository").
+ Relation("Repository.Tags", func(q *bun.SelectQuery) *bun.SelectQuery {
+ return q.Where("category = ?", "evaluation")
+ }).
+ Where("path IN (?)", bun.In(paths)).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find models by path,error: %w", err)
+ }
+
+ var sortedDatasets []Dataset
+ for _, path := range paths {
+ for _, ds := range datasets {
+ if ds.Repository.Path == path {
+ sortedDatasets = append(sortedDatasets, ds)
+ }
+ }
+ }
+
+ datasets = nil
+ return sortedDatasets, nil
+}
+
+func (s *datasetStoreImpl) CreateIfNotExist(ctx context.Context, input Dataset) (*Dataset, error) {
+ err := s.db.Core.NewSelect().
+ Model(&input).
+ Where("repository_id = ?", input.RepositoryID).
+ Relation("Repository").
+ Scan(ctx)
+ if err == nil {
+ return &input, nil
+ }
+
+ input.LastUpdatedAt = time.Now()
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create dataset in db failed", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("create dataset in db failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log"
+ "strings"
+ "time"
+
+ "github.com/uptrace/bun/dialect/pgdialect"
+ "github.com/uptrace/bun/dialect/sqlitedialect"
+ "github.com/uptrace/bun/driver/pgdriver"
+ "github.com/uptrace/bun/driver/sqliteshim"
+ "github.com/uptrace/bun/extra/bundebug"
+
+ "github.com/uptrace/bun"
+)
+
+type DatabaseDialect string
+
+// default database connection shared between different stores
+var defaultDB *DB
+
+const (
+ DialectPostgres DatabaseDialect = "pg"
+ DialectSQLite DatabaseDialect = "sqlite"
+)
+
+// DBConfig config of database
+type DBConfig struct {
+ // database vendor to use, valid value: pg, sqlite
+ Dialect DatabaseDialect `json:"dialect" comment:"database vendor to use, valid value: pg, sqlite"`
+ // e.g.: postgresql://starhub:starhub@localhost:5433/starhub?sslmode=disable
+ DSN string `json:"dsn" comment:"e.g.: postgresql://starhub:starhub@localhost:5433/starhub?sslmode=disable"`
+}
+
+// DB is where all database operation lives
+type DB struct {
+ // Operator is where database access/write methods implemented
+ Operator
+
+ // the underlying *bun.DB
+ BunDB *bun.DB
+}
+
+// Operator is where database access/write methods implemented
+// so that we don't have to write the same method on both DB and Transaction.
+type Operator struct {
+ Core bun.IDB
+}
+
+func InitDB(config DBConfig) {
+ bg, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ var err error
+ defaultDB, err = NewDB(bg, config)
+ if err != nil {
+ log.Fatal("failed to initialize database", err.Error())
+ }
+}
+
+// NewDB initializes a DB via config
+func NewDB(ctx context.Context, config DBConfig) (db *DB, err error) {
+ var bunDB *bun.DB
+
+ switch config.Dialect {
+ case DialectPostgres:
+ sqlDB := sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(config.DSN)))
+ bunDB = bun.NewDB(sqlDB, pgdialect.New(), bun.WithDiscardUnknownColumns())
+ case DialectSQLite:
+ var sqlDB *sql.DB
+ sqlDB, err = sql.Open(sqliteshim.ShimName, config.DSN)
+ if err != nil {
+ err = fmt.Errorf("sql.Open: %w", err)
+ return
+ }
+
+ // in-memory database is deleted when the connection is closed.
+ // we should avoid this.
+ // ref: https://www.sqlite.org/inmemorydb.html
+ if strings.Contains(config.DSN, ":memory:") ||
+ strings.Contains(config.DSN, "mode=memory") {
+ sqlDB.SetMaxIdleConns(100)
+ sqlDB.SetConnMaxLifetime(0)
+ }
+
+ // SQLite allows multiple readers but only a single writer at any one time.
+ // If you have multiple connections to the same DB, you will inevitably run into database is locked.
+ // This is an unavoidable consequence of sqlite's locking model,
+ // and therefore something your application needs to deal with.
+ // ref: https://github.com/mattn/go-sqlite3/issues/274#issuecomment-232942571
+ //
+ // Tip: Don't write through DB and TX at the same time.
+ bunDB = bun.NewDB(sqlDB, sqlitedialect.New(), bun.WithDiscardUnknownColumns())
+ default:
+ err = fmt.Errorf("unknown database dialect %q", config.Dialect)
+ return
+ }
+
+ bunDB.AddQueryHook(bundebug.NewQueryHook(
+ bundebug.FromEnv("DB_DEBUG"),
+ ))
+
+ err = bunDB.PingContext(ctx)
+ if err != nil {
+ err = fmt.Errorf("pinging %s database: %w", config.Dialect, err)
+ return
+ }
+
+ db = &DB{
+ Operator: Operator{Core: bunDB},
+ BunDB: bunDB,
+ }
+
+ db.BunDB.RegisterModel((*RepositoryTag)(nil))
+ db.BunDB.RegisterModel((*CollectionRepository)(nil))
+ return
+}
+
+// RunInTx runs the function in a transaction.
+// If the function returns an error, the transaction is rolled back.
+// Otherwise, the transaction is committed.
+func (db *DB) RunInTx(ctx context.Context, fn func(ctx context.Context, tx Operator) error) error {
+ return db.BunDB.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, tx bun.Tx) error {
+ op := Operator{Core: tx}
+ return fn(ctx, op)
+ })
+}
+
+// Close closes the database and prevents new queries from starting.
+// Close then waits for all queries that have started processing on the server to finish.
+// It is rare to Close a DB,
+// as the DB handle is meant to be long-lived and shared between many goroutines.
+func (db *DB) Close() error {
+ return db.BunDB.Close()
+}
+
+
+
package database
+
+import "github.com/uptrace/bun"
+
+type SelectOption interface {
+ Appply(query *bun.SelectQuery)
+}
+
+type ColumnOption struct {
+ cols []string
+}
+
+// Appply implements SelectOption.
+func (c *ColumnOption) Appply(q *bun.SelectQuery) {
+ q.Column(c.cols...)
+}
+
+func Columns(columns ...string) SelectOption {
+ return &ColumnOption{
+ cols: columns,
+ }
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type Deploy struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ // space_id to deploy, it's 0 if deploy model
+ SpaceID int64 `bun:",notnull" json:"space_id"`
+ Status int `bun:",notnull" json:"status"`
+ GitPath string `bun:",notnull" json:"git_path"`
+ GitBranch string `bun:",notnull" json:"git_branch"`
+ Env string `bun:",nullzero" json:"env"`
+ Secret string `bun:",nullzero" json:"secret"`
+ Template string `bun:",notnull" json:"tmeplate"`
+ Hardware string `bun:",notnull" json:"hardware"`
+ // for image run task, aka task_type = 1
+ // running image of cluster, comes from builder or pre-define
+ ImageID string `bun:",nullzero" json:"image_id"`
+ // add at 2024-05
+ DeployName string `json:"deploy_name"`
+ // user_id trigger deploy action, rather than repo owner user_id
+ UserID int64 `json:"user_id"`
+ // model_id to deploy, it's 0 if deploy space
+ ModelID int64 `json:"model_id"`
+ // repository_id of model/space/code/dataset
+ RepoID int64 `json:"repo_id"`
+ // model running engine vllm or TGI
+ RuntimeFramework string `bun:",nullzero" json:"runtime_framework"`
+ ContainerPort int `json:"container_port"`
+ Annotation string `bun:",nullzero" json:"annotation"`
+ MinReplica int `json:"min_replica"`
+ MaxReplica int `json:"max_replica"`
+ SvcName string `json:"svc_name"`
+ Endpoint string `json:"endpoint"`
+ ClusterID string `json:"cluster_id"`
+ SecureLevel int `json:"secure_level"` // 1-public, 2-private, 3-extension in future
+ Type int `json:"type"` // 0-space, 1-inference, 2-finetune, 3-serverless
+ UserUUID string `bun:"," json:"user_uuid"`
+ SKU string `bun:"," json:"sku"`
+ OrderDetailID int64 `bun:"," json:"order_detail_id"`
+ times
+}
+
+type DeployTask struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ // 0: build, 1: run
+ TaskType int `bun:",notnull" json:"task_type"`
+ Status int `bun:",notnull" json:"status"`
+ Message string `bun:",nullzero" json:"message"`
+ DeployID int64 `bun:",notnull" json:"deploy_id"`
+ Deploy *Deploy `bun:"rel:belongs-to,join:deploy_id=id" json:"deploy"`
+ times
+}
+
+type deployTaskStoreImpl struct {
+ db *DB
+}
+
+type DeployTaskStore interface {
+ CreateDeploy(ctx context.Context, deploy *Deploy) error
+ UpdateDeploy(ctx context.Context, deploy *Deploy) error
+ GetLatestDeployBySpaceID(ctx context.Context, spaceID int64) (*Deploy, error)
+ CreateDeployTask(ctx context.Context, deployTask *DeployTask) error
+ UpdateDeployTask(ctx context.Context, deployTask *DeployTask) error
+ GetDeployTask(ctx context.Context, id int64) (*DeployTask, error)
+ GetDeployTasksOfDeploy(ctx context.Context, deployID int64) ([]*DeployTask, error)
+ // GetNewTaskAfter return the first task of the next deploy
+ GetNewTaskAfter(ctx context.Context, currentDeployTaskID int64) (*DeployTask, error)
+ // GetNewTaskFirst returns the first task which has not end
+ GetNewTaskFirst(ctx context.Context) (*DeployTask, error)
+ UpdateInTx(ctx context.Context, deployColumns, deployTaskColumns []string, deploy *Deploy, deployTasks ...*DeployTask) error
+ ListDeploy(ctx context.Context, repoType types.RepositoryType, repoID, userID int64) ([]Deploy, error)
+ DeleteDeploy(ctx context.Context, repoType types.RepositoryType, repoID, userID int64, deployID int64) error
+ ListDeployByUserID(ctx context.Context, userID int64, req *types.DeployReq) ([]Deploy, int, error)
+ ListInstancesByUserID(ctx context.Context, userID int64, per, page int) ([]Deploy, int, error)
+ GetDeployByID(ctx context.Context, deployID int64) (*Deploy, error)
+ GetDeployBySvcName(ctx context.Context, svcName string) (*Deploy, error)
+ StopDeploy(ctx context.Context, repoType types.RepositoryType, repoID, userID int64, deployID int64) error
+ GetServerlessDeployByRepID(ctx context.Context, repoID int64) (*Deploy, error)
+ ListServerless(ctx context.Context, req types.DeployReq) ([]Deploy, int, error)
+ GetRunningInferenceAndFinetuneByUserID(ctx context.Context, userID int64) ([]Deploy, error)
+ ListAllDeployments(ctx context.Context, userID int64) ([]Deploy, error)
+}
+
+func NewDeployTaskStore() DeployTaskStore {
+ return &deployTaskStoreImpl{db: defaultDB}
+}
+
+func NewDeployTaskStoreWithDB(db *DB) DeployTaskStore {
+ return &deployTaskStoreImpl{db: db}
+}
+
+func (s *deployTaskStoreImpl) CreateDeploy(ctx context.Context, deploy *Deploy) error {
+ _, err := s.db.Core.NewInsert().Model(deploy).Exec(ctx, deploy)
+ return err
+}
+
+func (s *deployTaskStoreImpl) UpdateDeploy(ctx context.Context, deploy *Deploy) error {
+ _, err := s.db.Core.NewUpdate().Model(deploy).WherePK().Exec(ctx)
+ return err
+}
+
+func (s *deployTaskStoreImpl) GetLatestDeployBySpaceID(ctx context.Context, spaceID int64) (*Deploy, error) {
+ deploy := &Deploy{}
+ err := s.db.Core.NewSelect().Model(deploy).Where("space_id = ?", spaceID).Order("created_at DESC").Limit(1).Scan(ctx, deploy)
+ return deploy, err
+}
+
+func (s *deployTaskStoreImpl) CreateDeployTask(ctx context.Context, deployTask *DeployTask) error {
+ _, err := s.db.Core.NewInsert().Model(deployTask).Exec(ctx, deployTask)
+ return err
+}
+
+func (s *deployTaskStoreImpl) UpdateDeployTask(ctx context.Context, deployTask *DeployTask) error {
+ _, err := s.db.Core.NewUpdate().Model(deployTask).WherePK().Exec(ctx)
+ return err
+}
+
+func (s *deployTaskStoreImpl) GetDeployTask(ctx context.Context, id int64) (*DeployTask, error) {
+ deployTask := &DeployTask{}
+ err := s.db.Core.NewSelect().Model(deployTask).Where("deploy_task.id = ?", id).
+ Relation("Deploy").
+ Limit(1).
+ Scan(ctx, deployTask)
+ return deployTask, err
+}
+
+func (s *deployTaskStoreImpl) GetDeployTasksOfDeploy(ctx context.Context, deployID int64) ([]*DeployTask, error) {
+ var deployTasks []*DeployTask
+ err := s.db.Core.NewSelect().Model((*DeployTask)(nil)).Where("deploy_id = ?", deployID).Scan(ctx, &deployTasks)
+ return deployTasks, err
+}
+
+// GetNewTaskAfter return the first task of the next deploy
+func (s *deployTaskStoreImpl) GetNewTaskAfter(ctx context.Context, currentDeployTaskID int64) (*DeployTask, error) {
+ deployTask := &DeployTask{}
+ err := s.db.Core.NewSelect().Model(deployTask).Relation("Deploy").
+ Where("deploy_task.id > ? ", currentDeployTaskID).
+ Where("(task_type = 0 and deploy_task.status in (0,1)) or (task_type = 1 and deploy_task.status in (0,1,3))").
+ Order("id ASC").
+ Limit(1).
+ Scan(ctx, deployTask)
+ return deployTask, err
+}
+
+// GetNewTaskFirst returns the first task which has not end
+func (s *deployTaskStoreImpl) GetNewTaskFirst(ctx context.Context) (*DeployTask, error) {
+ deployTask := &DeployTask{}
+ err := s.db.Core.NewSelect().Model(deployTask).Relation("Deploy").
+ Where("(task_type = 0 and deploy_task.status in (0,1)) or (task_type = 1 and deploy_task.status in (0,1,3))").
+ Order("id asc").
+ Limit(1).
+ Scan(ctx, deployTask)
+ return deployTask, err
+}
+
+func (s *deployTaskStoreImpl) UpdateInTx(ctx context.Context, deployColumns, deployTaskColumns []string, deploy *Deploy, deployTasks ...*DeployTask) error {
+ tx, err := s.db.Core.BeginTx(ctx, nil)
+ if err != nil {
+ return fmt.Errorf("failed to begin transaction,%w", err)
+ }
+
+ if deploy != nil {
+ deployColumns = append(deployColumns, "updated_at")
+ deploy.UpdatedAt = time.Now()
+ _, err = tx.NewUpdate().Model(deploy).
+ Column(deployColumns...).
+ WherePK().Exec(ctx)
+ if err != nil {
+ if ree := tx.Rollback(); ree != nil {
+ slog.Error("rollback failed", "error", err)
+ }
+ return fmt.Errorf("failed to update deploy,%w", err)
+ }
+ }
+
+ for _, t := range deployTasks {
+ t.UpdatedAt = time.Now()
+ }
+ deployTaskColumns = append(deployTaskColumns, "updated_at")
+ _, err = tx.NewUpdate().
+ Model(&deployTasks).
+ // Column("status", "message", "updated_at").
+ Column(deployTaskColumns...).
+ Bulk().
+ Exec(ctx)
+ if err != nil {
+ if ree := tx.Rollback(); ree != nil {
+ slog.Error("rollback failed", "error", err)
+ }
+ return fmt.Errorf("failed to update deploy tasks in tx,%w", err)
+ }
+
+ return tx.Commit()
+}
+
+func (s *deployTaskStoreImpl) ListDeploy(ctx context.Context, repoType types.RepositoryType, repoID, userID int64) ([]Deploy, error) {
+ var result []Deploy
+ query := s.db.Operator.Core.NewSelect().Model(&result).Where("user_id = ? and repo_id = ?", userID, repoID)
+ if repoType == types.ModelRepo {
+ query = query.Where("status != ?", common.Deleted)
+ }
+ query = query.Order("id desc")
+ if repoType == types.SpaceRepo {
+ query = query.Limit(1)
+ }
+ _, err := query.Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func (s *deployTaskStoreImpl) DeleteDeploy(ctx context.Context, repoType types.RepositoryType, repoID, userID int64, deployID int64) error {
+ // only delete the deploy of specific repo was triggered by current login user
+ res, err := s.db.BunDB.Exec("Update deploys set status = ? where id = ? and repo_id = ? and user_id = ?", common.Deleted, deployID, repoID, userID)
+ if err != nil {
+ return err
+ }
+ err = assertAffectedOneRow(res, err)
+ return err
+}
+
+func (s *deployTaskStoreImpl) ListDeployByUserID(ctx context.Context, userID int64, req *types.DeployReq) ([]Deploy, int, error) {
+ var result []Deploy
+ query := s.db.Operator.Core.NewSelect().Model(&result).Where("user_id = ? and type = ?", userID, req.DeployType)
+ if req.RepoType == types.ModelRepo {
+ query = query.Where("model_id > 0 and status != ? ", common.Deleted)
+ }
+ query = query.Order("id desc")
+ if req.RepoType == types.SpaceRepo {
+ query = query.Where("space_id > 0")
+ query = query.Limit(1)
+ }
+ query = query.Limit(req.PageSize).Offset((req.Page - 1) * req.PageSize)
+ _, err := query.Exec(ctx, &result)
+ if err != nil {
+ return nil, 0, err
+ }
+ total, err := query.Count(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ return result, total, nil
+}
+
+func (s *deployTaskStoreImpl) ListInstancesByUserID(ctx context.Context, userID int64, per, page int) ([]Deploy, int, error) {
+ var result []Deploy
+ query := s.db.Operator.Core.NewSelect().Model(&result).Where("user_id = ?", userID)
+ query = query.Where("type = ? and status != ?", types.FinetuneType, common.Deleted)
+ query = query.Order("id desc")
+ query = query.Limit(per).Offset((page - 1) * per)
+ _, err := query.Exec(ctx, &result)
+ if err != nil {
+ return nil, 0, err
+ }
+ total, err := query.Count(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ return result, total, nil
+}
+
+func (s *deployTaskStoreImpl) GetDeployByID(ctx context.Context, deployID int64) (*Deploy, error) {
+ deploy := &Deploy{}
+ err := s.db.Operator.Core.NewSelect().Model(deploy).Where("id = ?", deployID).Scan(ctx, deploy)
+ return deploy, err
+}
+
+func (s *deployTaskStoreImpl) GetDeployBySvcName(ctx context.Context, svcName string) (*Deploy, error) {
+ deploy := &Deploy{}
+ err := s.db.Operator.Core.NewSelect().Model(deploy).Where("svc_name = ?", svcName).Scan(ctx, deploy)
+ return deploy, err
+}
+
+func (s *deployTaskStoreImpl) StopDeploy(ctx context.Context, repoType types.RepositoryType, repoID, userID int64, deployID int64) error {
+ // only stop the deploy of specific repo was triggered by current login user
+ res, err := s.db.BunDB.Exec("Update deploys set status=?,updated_at=current_timestamp where id = ? and repo_id = ? and user_id = ?", common.Stopped, deployID, repoID, userID)
+ if err != nil {
+ return err
+ }
+ err = assertAffectedOneRow(res, err)
+ return err
+}
+
+func (s *deployTaskStoreImpl) GetServerlessDeployByRepID(ctx context.Context, repoID int64) (*Deploy, error) {
+ deploy := &Deploy{}
+ err := s.db.Operator.Core.NewSelect().Model(deploy).Where("repo_id = ? and type = ?", repoID, types.ServerlessType).Scan(ctx, deploy)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("select serverless deploy error, %w", err)
+ }
+ return deploy, nil
+}
+
+func (s *deployTaskStoreImpl) ListServerless(ctx context.Context, req types.DeployReq) ([]Deploy, int, error) {
+ var result []Deploy
+ query := s.db.Operator.Core.NewSelect().Model(&result).Where("type = ?", req.DeployType)
+ query = query.Limit(req.PageSize).Offset((req.Page - 1) * req.PageSize)
+ _, err := query.Exec(ctx, &result)
+ if err != nil {
+ return nil, 0, err
+ }
+ total, err := query.Count(ctx)
+ if err != nil {
+ return nil, 0, err
+ }
+ return result, total, nil
+}
+
+func (s *deployTaskStoreImpl) GetRunningInferenceAndFinetuneByUserID(ctx context.Context, userID int64) ([]Deploy, error) {
+ // get all running inference and finetune of user
+ var result []Deploy
+ _, err := s.db.Operator.Core.NewSelect().Model(&result).
+ Where("user_id = ?", userID).
+ Where("type in (1,2)").
+ Where("status = ?", common.Running).
+ Exec(ctx, &result)
+ if err != nil && errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("select deploy error, %w", err)
+ }
+ return result, nil
+}
+
+func (s *deployTaskStoreImpl) ListAllDeployments(ctx context.Context, userID int64) ([]Deploy, error) {
+ var result []Deploy
+ err := s.db.Operator.Core.NewSelect().
+ Model(&result).
+ Where("user_id = ? and status IN (?)", userID, bun.In([]int64{common.Running, common.Deploying, common.Building})).
+ Scan(ctx)
+
+ return result, err
+}
+
+
+
package database
+
+import "context"
+
+type Discussion struct {
+ ID int64 `bun:"id,pk,autoincrement"`
+ UserID int64 `bun:"user_id,notnull"`
+ User *User `bun:"rel:belongs-to,join:user_id=id"`
+ Title string `bun:"title,notnull"`
+ DiscussionableID int64 `bun:"discussionable_id,notnull"`
+ DiscussionableType string `bun:"discussionable_type,notnull"`
+ CommentCount int64 `bun:"comment_count,notnull,default:0"`
+ times
+}
+
+type Comment struct {
+ ID int64 `bun:"id,pk,autoincrement"`
+ Content string `bun:"content"`
+ CommentableType string `bun:"commentable_type,notnull"`
+ CommentableID int64 `bun:"commentable_id,notnull"`
+ UserID int64 `bun:"user_id,notnull"`
+ User *User `bun:"rel:belongs-to,join:user_id=id"`
+ times
+}
+
+const (
+ CommentableTypeDiscussion = "discussion"
+ CommentableTypeArticle = "article"
+)
+
+const (
+ DiscussionableTypeRepo = "repo"
+ DiscussionableTypeCollection = "collection"
+)
+
+type discussionStoreImpl struct {
+ db *DB
+}
+
+type DiscussionStore interface {
+ Create(ctx context.Context, discussion Discussion) (*Discussion, error)
+ FindByID(ctx context.Context, id int64) (*Discussion, error)
+ FindByDiscussionableID(ctx context.Context, discussionableType string, discussionableID int64) ([]Discussion, error)
+ UpdateByID(ctx context.Context, id int64, title string) error
+ DeleteByID(ctx context.Context, id int64) error
+ FindDiscussionComments(ctx context.Context, discussionID int64) ([]Comment, error)
+ CreateComment(ctx context.Context, comment Comment) (*Comment, error)
+ UpdateComment(ctx context.Context, id int64, content string) error
+ FindCommentByID(ctx context.Context, id int64) (*Comment, error)
+ DeleteComment(ctx context.Context, id int64) error
+}
+
+func NewDiscussionStore() DiscussionStore {
+ return &discussionStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewDiscussionStoreWithDB(db *DB) DiscussionStore {
+ return &discussionStoreImpl{
+ db: db,
+ }
+}
+
+func (s *discussionStoreImpl) Create(ctx context.Context, discussion Discussion) (*Discussion, error) {
+ _, err := s.db.Core.NewInsert().Model(&discussion).Exec(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &discussion, nil
+}
+
+func (s *discussionStoreImpl) FindByID(ctx context.Context, id int64) (*Discussion, error) {
+ discussion := Discussion{}
+ err := s.db.Core.NewSelect().Model(&discussion).
+ Where("discussion.id = ?", id).
+ Relation("User").
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &discussion, nil
+}
+
+func (s *discussionStoreImpl) FindByDiscussionableID(ctx context.Context, discussionableType string, discussionableID int64) ([]Discussion, error) {
+ discussions := []Discussion{}
+ err := s.db.Core.NewSelect().Model(&discussions).
+ Where("discussionable_type = ? AND discussionable_id = ?", discussionableType, discussionableID).
+ Relation("User").
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return discussions, nil
+}
+
+func (s *discussionStoreImpl) UpdateByID(ctx context.Context, id int64, title string) error {
+ _, err := s.db.Core.NewUpdate().Model(&Discussion{}).Set("title = ?", title).Where("id = ?", id).Exec(ctx)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *discussionStoreImpl) DeleteByID(ctx context.Context, id int64) error {
+ _, err := s.db.Core.NewDelete().Model(&Discussion{}).Where("id = ?", id).Exec(ctx)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *discussionStoreImpl) FindDiscussionComments(ctx context.Context, discussionID int64) ([]Comment, error) {
+ comments := []Comment{}
+ err := s.db.Core.NewSelect().Model(&comments).
+ Relation("User").
+ Where("commentable_type=? AND commentable_id = ?", CommentableTypeDiscussion, discussionID).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return comments, nil
+}
+
+func (s *discussionStoreImpl) CreateComment(ctx context.Context, comment Comment) (*Comment, error) {
+ _, err := s.db.Core.NewInsert().Model(&comment).Exec(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &comment, nil
+}
+
+func (s *discussionStoreImpl) UpdateComment(ctx context.Context, id int64, content string) error {
+ _, err := s.db.Core.NewUpdate().Model(&Comment{}).Set("content = ?", content).Where("id = ?", id).Exec(ctx)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+func (s *discussionStoreImpl) FindCommentByID(ctx context.Context, id int64) (*Comment, error) {
+ comment := Comment{}
+ err := s.db.Core.NewSelect().Model(&comment).
+ Where("comment.id = ?", id).
+ Relation("User").
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &comment, nil
+}
+
+func (s *discussionStoreImpl) DeleteComment(ctx context.Context, id int64) error {
+ _, err := s.db.Core.NewDelete().Model(&Comment{}).Where("id = ?", id).Exec(ctx)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+
+
package database
+
+import "context"
+
+type eventStoreImpl struct {
+ db *DB
+}
+
+type EventStore interface {
+ Save(ctx context.Context, event Event) error
+ // batch insert
+ BatchSave(ctx context.Context, events []Event) error
+}
+
+func NewEventStore() EventStore {
+ return &eventStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewEventStoreWithDB(db *DB) EventStore {
+ return &eventStoreImpl{
+ db: db,
+ }
+}
+
+func (s *eventStoreImpl) Save(ctx context.Context, event Event) error {
+ return assertAffectedOneRow(s.db.Core.NewInsert().Model(&event).Exec(ctx))
+}
+
+// batch insert
+func (s *eventStoreImpl) BatchSave(ctx context.Context, events []Event) error {
+ result, err := s.db.Core.NewInsert().Model(&events).Exec(ctx)
+ return assertAffectedXRows(int64(len(events)), result, err)
+}
+
+
+
package database
+
+import (
+ "context"
+)
+
+type fileStoreImpl struct {
+ db *DB
+}
+
+type FileStore interface {
+ FindByParentPath(ctx context.Context, repoID int64, path string) ([]File, error)
+ BatchCreate(ctx context.Context, files []File) error
+}
+
+func NewFileStoreWithDB(db *DB) FileStore {
+ return &fileStoreImpl{
+ db: db,
+ }
+}
+
+func NewFileStore() FileStore {
+ return &fileStoreImpl{
+ db: defaultDB,
+ }
+}
+
+// File represents a file in a repository, *only used by multi-sync client*
+type File struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `json:"name"`
+ Path string `json:"path"`
+ ParentPath string `json:"parent_path"`
+ Size int64 `json:"size"`
+ LastCommitMessage string `json:"last_commit_message"`
+ LastCommitDate string `json:"last_commit_date"`
+ RepositoryID int64 `json:"repository_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ times
+}
+
+func (s *fileStoreImpl) FindByParentPath(ctx context.Context, repoID int64, path string) ([]File, error) {
+ var files []File
+ err := s.db.Operator.Core.NewSelect().
+ Model(&files).
+ Where("parent_path = ? and repository_id = ?", path, repoID).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return files, nil
+}
+
+func (s *fileStoreImpl) BatchCreate(ctx context.Context, files []File) error {
+ result, err := s.db.Operator.Core.NewInsert().
+ Model(&files).
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return assertAffectedXRows(int64(len(files)), result, err)
+}
+
+
+
package database
+
+import "context"
+
+type gitServerAccessTokenStoreImpl struct {
+ db *DB
+}
+
+type GitServerAccessTokenStore interface {
+ Create(ctx context.Context, gToken *GitServerAccessToken) (*GitServerAccessToken, error)
+ Index(ctx context.Context) ([]GitServerAccessToken, error)
+ FindByType(ctx context.Context, serverType string) ([]GitServerAccessToken, error)
+}
+
+func NewGitServerAccessTokenStoreWithDB(db *DB) GitServerAccessTokenStore {
+ return &gitServerAccessTokenStoreImpl{
+ db: db,
+ }
+}
+
+func NewGitServerAccessTokenStore() GitServerAccessTokenStore {
+ return &gitServerAccessTokenStoreImpl{
+ db: defaultDB,
+ }
+}
+
+type GitServerType string
+
+const (
+ MirrorServer GitServerType = "mirror"
+ GitServer GitServerType = "git"
+)
+
+type GitServerAccessToken struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Token string `bun:",notnull" json:"token"`
+ ServerType GitServerType `bun:",notnull" json:"server_type"`
+ times
+}
+
+func (s *gitServerAccessTokenStoreImpl) Create(ctx context.Context, gToken *GitServerAccessToken) (*GitServerAccessToken, error) {
+ err := s.db.Operator.Core.NewInsert().
+ Model(gToken).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return gToken, nil
+}
+
+func (s *gitServerAccessTokenStoreImpl) Index(ctx context.Context) ([]GitServerAccessToken, error) {
+ var gTokens []GitServerAccessToken
+ err := s.db.Operator.Core.NewSelect().
+ Model(&gTokens).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return gTokens, nil
+}
+
+func (s *gitServerAccessTokenStoreImpl) FindByType(ctx context.Context, serverType string) ([]GitServerAccessToken, error) {
+ var gTokens []GitServerAccessToken
+ err := s.db.Operator.Core.NewSelect().
+ Model(&gTokens).
+ Where("server_type = ?", serverType).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return gTokens, nil
+}
+
+
+
package database
+
+import (
+ "context"
+)
+
+type lfsLockStoreImpl struct {
+ db *DB
+}
+
+type LfsLockStore interface {
+ FindByID(ctx context.Context, ID int64) (*LfsLock, error)
+ FindByPath(ctx context.Context, RepoId int64, path string) (*LfsLock, error)
+ FindByRepoID(ctx context.Context, RepoId int64, page, per int) ([]LfsLock, error)
+ Create(ctx context.Context, lfsLock LfsLock) (*LfsLock, error)
+ RemoveByID(ctx context.Context, ID int64) error
+}
+
+func NewLfsLockStore() LfsLockStore {
+ return &lfsLockStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewLfsLockStoreWithDB(db *DB) LfsLockStore {
+ return &lfsLockStoreImpl{
+ db: db,
+ }
+}
+
+type LfsLock struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ Path string `bun:",notnull" json:"path"`
+ times
+}
+
+func (s *lfsLockStoreImpl) FindByID(ctx context.Context, ID int64) (*LfsLock, error) {
+ var lfsLock LfsLock
+ err := s.db.Operator.Core.NewSelect().
+ Model(&lfsLock).
+ Relation("User").
+ Where("lfs_lock.id = ?", ID).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &lfsLock, nil
+}
+
+func (s *lfsLockStoreImpl) FindByPath(ctx context.Context, RepoId int64, path string) (*LfsLock, error) {
+ var lfsLock LfsLock
+ err := s.db.Operator.Core.NewSelect().
+ Model(&lfsLock).
+ Relation("User").
+ Where("path=? and repository_id=?", path, RepoId).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &lfsLock, nil
+}
+
+func (s *lfsLockStoreImpl) FindByRepoID(ctx context.Context, RepoId int64, page, per int) ([]LfsLock, error) {
+ var lfsLocks []LfsLock
+ query := s.db.Operator.Core.NewSelect().
+ Model(&lfsLocks).
+ Relation("User").
+ Where("repository_id=?", RepoId)
+
+ if page > 0 && per > 0 {
+ query = query.Limit(per).Offset((page - 1) * per)
+ }
+ err := query.Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return lfsLocks, nil
+}
+
+func (s *lfsLockStoreImpl) Create(ctx context.Context, lfsLock LfsLock) (*LfsLock, error) {
+ err := s.db.Operator.Core.NewInsert().
+ Model(&lfsLock).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &lfsLock, nil
+}
+
+func (s *lfsLockStoreImpl) RemoveByID(ctx context.Context, ID int64) error {
+ _, err := s.db.Operator.Core.NewDelete().
+ Model(&LfsLock{}).
+ Where("id = ?", ID).
+ Exec(ctx)
+
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "time"
+)
+
+type lfsMetaObjectStoreImpl struct {
+ db *DB
+}
+
+type LfsMetaObjectStore interface {
+ FindByOID(ctx context.Context, RepoId int64, Oid string) (*LfsMetaObject, error)
+ FindByRepoID(ctx context.Context, repoID int64) ([]LfsMetaObject, error)
+ Create(ctx context.Context, lfsObj LfsMetaObject) (*LfsMetaObject, error)
+ RemoveByOid(ctx context.Context, oid string, repoID int64) error
+ UpdateOrCreate(ctx context.Context, input LfsMetaObject) (*LfsMetaObject, error)
+ BulkUpdateOrCreate(ctx context.Context, input []LfsMetaObject) error
+}
+
+func NewLfsMetaObjectStore() LfsMetaObjectStore {
+ return &lfsMetaObjectStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewLfsMetaObjectStoreWithDB(db *DB) LfsMetaObjectStore {
+ return &lfsMetaObjectStoreImpl{
+ db: db,
+ }
+}
+
+type LfsMetaObject struct {
+ ID int64 `bun:",pk,autoincrement" json:"user_id"`
+ Oid string `bun:",notnull" json:"oid"`
+ Size int64 `bun:",notnull" json:"size"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ Existing bool `bun:",notnull" json:"existing"`
+ times
+}
+
+func (s *lfsMetaObjectStoreImpl) FindByOID(ctx context.Context, RepoId int64, Oid string) (*LfsMetaObject, error) {
+ var lfsMetaObject LfsMetaObject
+ err := s.db.Operator.Core.NewSelect().
+ Model(&lfsMetaObject).
+ Where("oid=? and repository_id=?", Oid, RepoId).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &lfsMetaObject, nil
+}
+
+func (s *lfsMetaObjectStoreImpl) FindByRepoID(ctx context.Context, repoID int64) ([]LfsMetaObject, error) {
+ var lfsMetaObjects []LfsMetaObject
+ err := s.db.Operator.Core.NewSelect().
+ Model(&lfsMetaObjects).
+ Where("repository_id=?", repoID).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return lfsMetaObjects, nil
+}
+
+func (s *lfsMetaObjectStoreImpl) Create(ctx context.Context, lfsObj LfsMetaObject) (*LfsMetaObject, error) {
+ err := s.db.Operator.Core.NewInsert().
+ Model(&lfsObj).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &lfsObj, nil
+}
+
+func (s *lfsMetaObjectStoreImpl) RemoveByOid(ctx context.Context, oid string, repoID int64) error {
+ _, err := s.db.Operator.Core.NewDelete().
+ Model(&LfsMetaObject{}).
+ Where("oid = ? and repository_id= ?", oid, repoID).
+ Exec(ctx)
+
+ return err
+}
+
+func (s *lfsMetaObjectStoreImpl) UpdateOrCreate(ctx context.Context, input LfsMetaObject) (*LfsMetaObject, error) {
+ input.UpdatedAt = time.Now()
+ _, err := s.db.Core.NewUpdate().
+ Model(&input).
+ Where("oid = ? and repository_id = ?", input.Oid, input.RepositoryID).
+ Returning("*").
+ Exec(ctx, &input)
+ if err == nil {
+ return &input, nil
+ }
+
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create lfs meta object in tx failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *lfsMetaObjectStoreImpl) BulkUpdateOrCreate(ctx context.Context, input []LfsMetaObject) error {
+ if len(input) == 0 {
+ return nil
+ }
+ _, err := s.db.Core.NewInsert().
+ Model(&input).
+ On("CONFLICT (oid, repository_id) DO UPDATE").
+ Set("size = EXCLUDED.size, updated_at = EXCLUDED.updated_at, existing = EXCLUDED.existing").
+ Exec(ctx)
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type LicenseStore interface {
+ List(ctx context.Context, req types.QueryLicenseReq) ([]License, int, error)
+ Create(ctx context.Context, input License) error
+ GetByID(ctx context.Context, id int64) (*License, error)
+ Update(ctx context.Context, input License) error
+ GetLatestActive(ctx context.Context) (*License, error)
+ Delete(ctx context.Context, input License) error
+}
+
+type licenseStoreImpl struct {
+ db *DB
+}
+
+type License struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Key string `bun:",notnull" json:"key"`
+ Company string `bun:",notnull" json:"company"`
+ Email string `bun:",notnull" json:"email"`
+ Product string `bun:",notnull" json:"product"`
+ Edition string `bun:",notnull" json:"edition"`
+ Version string `bun:",nullzero" json:"version"`
+ Status string `bun:",nullzero" json:"status"`
+ MaxUser int `bun:",notnull" json:"max_user"`
+ StartTime time.Time `bun:",notnull" json:"start_time"`
+ ExpireTime time.Time `bun:",notnull" json:"expire_time"`
+ Extra string `bun:",nullzero" json:"extra"`
+ Remark string `bun:",nullzero" json:"remark"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ times
+}
+
+func NewLicenseStore() LicenseStore {
+ return &licenseStoreImpl{db: defaultDB}
+}
+
+func NewLicenseStoreWithDB(db *DB) LicenseStore {
+ return &licenseStoreImpl{db: db}
+}
+
+func (s *licenseStoreImpl) List(ctx context.Context, req types.QueryLicenseReq) ([]License, int, error) {
+ var licenses []License
+ query := s.db.Operator.Core.NewSelect().Model(&licenses)
+
+ if req.Product != "" {
+ query = query.Where("product = ?", req.Product)
+ }
+
+ if req.Edition != "" {
+ query = query.Where("edition = ?", req.Edition)
+ }
+
+ if req.Search != "" {
+ query = query.Where("company LIKE ? OR email LIKE ? OR remark LIKE ?",
+ fmt.Sprintf("%%%s%%", req.Search),
+ fmt.Sprintf("%%%s%%", req.Search),
+ fmt.Sprintf("%%%s%%", req.Search))
+ }
+
+ total, err := query.Count(ctx)
+ if err != nil {
+ return nil, 0, fmt.Errorf("count license with error: %w", err)
+ }
+
+ query = query.Order("id DESC").Limit(req.Per).Offset((req.Page - 1) * req.Per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return nil, 0, fmt.Errorf("select license with error: %w", err)
+ }
+
+ return licenses, total, nil
+}
+
+// Create implements LicenseStoreInterface.
+func (s *licenseStoreImpl) Create(ctx context.Context, input License) error {
+ _, err := s.db.Operator.Core.NewInsert().Model(&input).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("insert license with error: %w", err)
+ }
+ return nil
+}
+
+// GetByID implements LicenseStoreInterface.
+func (s *licenseStoreImpl) GetByID(ctx context.Context, id int64) (*License, error) {
+ var license License
+ err := s.db.Operator.Core.NewSelect().Model(&license).Where("id = ?", id).Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select license by id %d with error: %w", id, err)
+ }
+ return &license, nil
+}
+
+func (s *licenseStoreImpl) Update(ctx context.Context, input License) error {
+ _, err := s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("update license with error: %w", err)
+ }
+ return nil
+}
+
+func (s *licenseStoreImpl) Delete(ctx context.Context, input License) error {
+ res, err := s.db.Operator.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("delete license failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *licenseStoreImpl) GetLatestActive(ctx context.Context) (*License, error) {
+ var license License
+ err := s.db.Operator.Core.NewSelect().Model(&license).
+ Where("start_time <= NOW() AND expire_time >= NOW()").
+ Order("id DESC").
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select latest active license with error: %w", err)
+ }
+ return &license, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type lLMConfigStoreImpl struct {
+ db *DB
+}
+
+type LLMConfig struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ModelName string `bun:",notnull" json:"model_name"`
+ ApiEndpoint string `bun:",notnull" json:"api_endpoint"`
+ AuthHeader string `bun:",notnull" json:"auth_header"`
+ Type int `bun:",notnull" json:"type"` // 1: optimization, 2: comparson
+ Enabled bool `bun:",notnull" json:"enabled"`
+ times
+}
+
+type LLMConfigStore interface {
+ GetOptimization(ctx context.Context) (*LLMConfig, error)
+}
+
+func NewLLMConfigStore() LLMConfigStore {
+ return &lLMConfigStoreImpl{db: defaultDB}
+}
+
+func NewLLMConfigStoreWithDB(db *DB) LLMConfigStore {
+ return &lLMConfigStoreImpl{db: db}
+}
+
+func (s *lLMConfigStoreImpl) GetOptimization(ctx context.Context) (*LLMConfig, error) {
+ var config LLMConfig
+ err := s.db.Operator.Core.NewSelect().Model(&config).Where("type = 1 and enabled = true").Limit(1).Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select optimization llm, %w", err)
+ }
+ return &config, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type memberStoreImpl struct {
+ db *DB
+}
+
+type MemberStore interface {
+ Find(ctx context.Context, orgID, userID int64) (*Member, error)
+ Add(ctx context.Context, orgID, userID int64, role string) error
+ Delete(ctx context.Context, orgID, userID int64, role string) error
+ UserMembers(ctx context.Context, userID int64) ([]Member, error)
+ OrganizationMembers(ctx context.Context, orgID int64, pageSize, page int) ([]Member, int, error)
+}
+
+func NewMemberStore() MemberStore {
+ return &memberStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewMemberStoreWithDB(db *DB) MemberStore {
+ return &memberStoreImpl{
+ db: db,
+ }
+}
+
+// Member is the relationship between a user and an organization.
+type Member struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ OrganizationID int64 `bun:",pk" json:"organization_id"`
+ UserID int64 `bun:",pk" json:"user_id"`
+ Organization *Organization `bun:"rel:belongs-to,join:organization_id=id" json:"organization"`
+ User *User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ Role string `bun:",notnull" json:"role"`
+ times
+}
+
+func (s *memberStoreImpl) Find(ctx context.Context, orgID, userID int64) (*Member, error) {
+ var member Member
+ err := s.db.Core.NewSelect().Model(&member).Where("organization_id = ? AND user_id = ?", orgID, userID).Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &member, nil
+}
+
+func (s *memberStoreImpl) Add(ctx context.Context, orgID, userID int64, role string) error {
+ member := &Member{
+ OrganizationID: orgID,
+ UserID: userID,
+ Role: role,
+ }
+ result, err := s.db.Core.NewInsert().Model(member).Exec(ctx)
+ if err != nil {
+ return err
+ }
+ return assertAffectedOneRow(result, err)
+}
+
+func (s *memberStoreImpl) Delete(ctx context.Context, orgID, userID int64, role string) error {
+ var member Member
+ _, err := s.db.Core.NewDelete().Model(&member).Where("organization_id=? and user_id=? and role=?", orgID, userID, role).Exec(ctx)
+ return err
+}
+
+func (s *memberStoreImpl) UserMembers(ctx context.Context, userID int64) ([]Member, error) {
+ var members []Member
+ err := s.db.Core.NewSelect().Model((*Member)(nil)).Where("user_id=?", userID).Scan(ctx, &members)
+ return members, err
+}
+
+func (s *memberStoreImpl) OrganizationMembers(ctx context.Context, orgID int64, pageSize, page int) ([]Member, int, error) {
+ var members []Member
+ var total int
+ q := s.db.Core.NewSelect().Model((*Member)(nil)).
+ Relation("User").
+ Where("organization_id=?", orgID).
+ Limit(pageSize).
+ Offset((page - 1) * pageSize)
+ err := q.Scan(ctx, &members)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to find org members,caused by:%w", err)
+ }
+ total, err = q.Count(ctx)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to count org members,caused by:%w", err)
+ }
+ return members, total, nil
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type Tag struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name" yaml:"name"`
+ Category string `bun:",notnull" json:"category" yaml:"category"`
+ Group string `bun:",notnull" json:"group" yaml:"group"`
+ Scope TagScope `bun:",notnull" json:"scope" yaml:"scope"`
+ BuiltIn bool `bun:",notnull" json:"built_in" yaml:"built_in"`
+ ShowName string `bun:"" json:"show_name" yaml:"show_name"`
+ times
+}
+
+type User struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ GitID int64 `bun:",notnull" json:"git_id"`
+ Name string `bun:",notnull" json:"name"`
+ Username string `bun:",notnull,unique" json:"username"`
+ Email string `bun:",notnull,unique" json:"email"`
+ Password string `bun:",notnull" json:"-"`
+ AccessTokens []database.AccessToken `bun:"rel:has-many,join:id=user_id"`
+ Namespaces []database.Namespace `bun:"rel:has-many,join:id=user_id" json:"namespace"`
+ times
+}
+
+type Dataset struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name"`
+ UrlSlug string `bun:",notnull" json:"nickname"`
+ Description string `bun:",nullzero" json:"description"`
+ Likes int64 `bun:",notnull" json:"likes"`
+ Downloads int64 `bun:",notnull" json:"downloads"`
+ Path string `bun:",notnull" json:"path"`
+ GitPath string `bun:",notnull" json:"git_path"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *database.Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ LastUpdatedAt time.Time `bun:",notnull" json:"last"`
+ Private bool `bun:",notnull" json:"private"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User *User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ times
+}
+
+type Model struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name"`
+ UrlSlug string `bun:",notnull" json:"nickname"`
+ Description string `bun:",nullzero" json:"description"`
+ Likes int64 `bun:",notnull" json:"likes"`
+ Downloads int64 `bun:",notnull" json:"downloads"`
+ Path string `bun:",notnull" json:"path"`
+ GitPath string `bun:",notnull" json:"git_path"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *database.Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ LastUpdatedAt time.Time `bun:",notnull" json:"last_updated_at"`
+ Private bool `bun:",notnull" json:"private"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User *User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ times
+}
+
+var baseModelTables = []any{
+ User{},
+ database.RepositoryTag{},
+ database.Repository{},
+ database.Namespace{},
+ Tag{},
+ database.TagCategory{},
+ Model{},
+ Dataset{},
+ database.LfsFile{},
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, baseModelTables...)
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, baseModelTables...)
+ })
+}
+
+type NamespaceType string
+
+const (
+ UserNamespace NamespaceType = "user"
+ OrgNamespace NamespaceType = "organization"
+)
+
+type RepositoryType string
+
+const (
+ ModelType RepositoryType = "model"
+ DatasetType RepositoryType = "dataset"
+)
+
+type TagScope string
+
+const (
+ ModelTagScope TagScope = "model"
+ DatasetTagScope TagScope = "dataset"
+)
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.AccessToken{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.AccessToken{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.SSHKey{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.SSHKey{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+var orgTables = []any{
+ database.Organization{},
+ database.Member{},
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, orgTables...)
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, orgTables...)
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/uptrace/bun"
+ "gopkg.in/yaml.v2"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type Tags struct {
+ Tags []database.Tag
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return db.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, db bun.Tx) (err error) {
+ // Read the seeds file of tags
+ currentDir, err := filepath.Abs(filepath.Dir("."))
+ if err != nil {
+ fmt.Println("Error getting current directory:", err)
+ return
+ }
+ yamlFilePath := filepath.Join(currentDir, "builder", "store", "database", "seeds", "tags.yml")
+ yamlFile, err := os.ReadFile(yamlFilePath)
+ if err != nil {
+ fmt.Println("Error reading YAML file:", err)
+ return
+ }
+
+ var tags Tags
+ err = yaml.Unmarshal(yamlFile, &tags)
+ if err != nil {
+ fmt.Println("Error parsing YAML:", err)
+ return
+ }
+
+ ts := tags.Tags
+ _, err = db.NewInsert().Model(&ts).Exec(ctx)
+ if err != nil {
+ fmt.Println("Error inserting data:", err)
+ return
+ }
+ fmt.Println("Insert data successfully")
+ return
+ })
+ }, func(ctx context.Context, db *bun.DB) error {
+ fmt.Print(" [down migration] ")
+ return nil
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/uptrace/bun"
+ "gopkg.in/yaml.v2"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type Categories struct {
+ Categories []database.TagCategory
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return db.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, db bun.Tx) (err error) {
+ // Read the seeds file of tags
+ currentDir, err := filepath.Abs(filepath.Dir("."))
+ if err != nil {
+ fmt.Println("Error getting current directory:", err)
+ return
+ }
+ yamlFilePath := filepath.Join(currentDir, "builder", "store", "database", "seeds", "tag_categories.yml")
+ yamlFile, err := os.ReadFile(yamlFilePath)
+ if err != nil {
+ fmt.Println("Error reading YAML file:", err)
+ return
+ }
+
+ var tagCategory Categories
+ err = yaml.Unmarshal(yamlFile, &tagCategory)
+ if err != nil {
+ fmt.Println("Error parsing YAML:", err)
+ return
+ }
+
+ tcs := tagCategory.Categories
+ _, err = db.NewInsert().Model(&tcs).Exec(ctx)
+ if err != nil {
+ fmt.Println("Error inserting data:", err)
+ return
+ }
+ fmt.Println("Insert data successfully")
+ return
+ })
+ }, func(context.Context, *bun.DB) error {
+ fmt.Print(" [down migration] ")
+ return nil
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.RepositoryDownload{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.RepositoryDownload{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.GitServerAccessToken{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.GitServerAccessToken{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type Space struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *database.Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ // gradio, streamlit, docker etc
+ Sdk string `bun:",notnull" json:"sdk"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, Space{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, Space{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "database/sql"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func getModels(ctx context.Context, db bun.Tx) (models []Model, err error) {
+ err = db.NewSelect().
+ Model(&models).
+ Relation("Repository").
+ Scan(ctx)
+ return
+}
+
+func getDatasets(ctx context.Context, db bun.Tx) (datasets []Dataset, err error) {
+ err = db.NewSelect().
+ Model(&datasets).
+ Relation("Repository").
+ Scan(ctx)
+ return
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return db.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, db bun.Tx) (err error) {
+ var repositories []*database.Repository
+ models, err := getModels(ctx, db)
+ if err != nil {
+ return
+ }
+ datasets, err := getDatasets(ctx, db)
+ if err != nil {
+ return
+ }
+
+ for _, model := range models {
+ repository := model.Repository
+ repository.Likes = model.Likes
+ repository.Nickname = model.UrlSlug
+ repository.DownloadCount = model.Downloads
+ repositories = append(repositories, repository)
+ }
+
+ for _, dataset := range datasets {
+ repository := dataset.Repository
+ repository.Likes = dataset.Likes
+ repository.Nickname = dataset.UrlSlug
+ repository.DownloadCount = dataset.Downloads
+ repositories = append(repositories, repository)
+ }
+ if len(repositories) == 0 {
+ return nil
+ }
+
+ values := db.NewValues(&repositories)
+
+ _, err = db.NewUpdate().
+ With("_data", values).
+ Model(&database.Repository{}).
+ TableExpr("_data").
+ Set("nickname = _data.nickname").
+ Set("download_count = _data.download_count").
+ Where("repository.id = _data.id").
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return nil
+ })
+ }, func(ctx context.Context, db *bun.DB) error {
+ return db.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, db bun.Tx) (err error) {
+ var resModels []*Model
+ var resDatasets []*Dataset
+ models, err := getModels(ctx, db)
+ if err != nil {
+ return
+ }
+ datasets, err := getDatasets(ctx, db)
+ if err != nil {
+ return
+ }
+
+ for _, model := range models {
+ repository := model.Repository
+ model.Name = repository.Name
+ model.Description = repository.Description
+ model.Path = repository.Path
+ model.GitPath = repository.GitPath
+ model.UserID = repository.UserID
+ model.Private = repository.Private
+ model.Likes = repository.Likes
+ model.UrlSlug = repository.Nickname
+ model.Downloads = repository.DownloadCount
+ resModels = append(resModels, &model)
+ }
+
+ for _, dataset := range datasets {
+ repository := dataset.Repository
+ dataset.Name = repository.Name
+ dataset.Description = repository.Description
+ dataset.Path = repository.Path
+ dataset.GitPath = repository.GitPath
+ dataset.UserID = repository.UserID
+ dataset.Private = repository.Private
+ dataset.Likes = repository.Likes
+ dataset.UrlSlug = repository.Nickname
+ dataset.Downloads = repository.DownloadCount
+ resDatasets = append(resDatasets, &dataset)
+ }
+
+ if len(resModels) > 0 {
+ values := db.NewValues(&resModels)
+
+ _, err = db.NewUpdate().
+ With("_data", values).
+ Model(&Model{}).
+ TableExpr("_data").
+ Set("name = _data.name").
+ Set("description = _data.description").
+ Set("path = _data.path").
+ Set("git_path = _data.git_path").
+ Set("user_id = _data.user_id").
+ Set("private = _data.private").
+ Set("likes = _data.likes").
+ Set("url_slug = _data.url_slug").
+ Set("downloads = _data.downloads").
+ Where("model.id = _data.id").
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ }
+
+ if len(resDatasets) > 0 {
+ values := db.NewValues(&resDatasets)
+
+ _, err = db.NewUpdate().
+ With("_data", values).
+ Model(&Dataset{}).
+ TableExpr("_data").
+ Set("name = _data.name").
+ Set("description = _data.description").
+ Set("path = _data.path").
+ Set("git_path = _data.git_path").
+ Set("user_id = _data.user_id").
+ Set("private = _data.private").
+ Set("likes = _data.likes").
+ Set("url_slug = _data.url_slug").
+ Set("downloads = _data.downloads").
+ Where("dataset.id = _data.id").
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.Code{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.Code{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.SpaceSdk{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.SpaceSdk{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.SpaceResource{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.SpaceResource{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type Deploy struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ // space_id to deploy, it's 0 if deploy model
+ SpaceID int64 `bun:",notnull" json:"space_id"`
+ Status int `bun:",notnull" json:"status"`
+ GitPath string `bun:",notnull" json:"git_path"`
+ GitBranch string `bun:",notnull" json:"git_branch"`
+ Env string `bun:",nullzero" json:"env"`
+ Secret string `bun:",nullzero" json:"secret"`
+ Template string `bun:",notnull" json:"tmeplate"`
+ Hardware string `bun:",notnull" json:"hardware"`
+ // for image run task, aka task_type = 1
+ // running image of cluster, comes from builder or pre-define
+ ImageID string `bun:",nullzero" json:"image_id"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ _ = dropTables(ctx, db, database.Space{})
+ return createTables(ctx, db, database.Space{}, Deploy{}, database.DeployTask{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, Deploy{}, database.DeployTask{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, database.RepoRelation{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*database.RepoRelation)(nil)).
+ Index("idx_repo_relation_from_repo_id").
+ Column("from_repo_id").
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*database.RepoRelation)(nil)).
+ Index("idx_repo_relation_to_repo_id").
+ Column("to_repo_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.RepoRelation{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, database.UserLike{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*database.UserLike)(nil)).
+ Index("idx_user_likes_user_id").
+ Column("user_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.UserLike{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ _ = dropTables(ctx, db, database.RecomOpWeight{}, database.RecomRepoScore{}, database.RecomWeight{})
+ err := createTables(ctx, db, database.RecomOpWeight{}, database.RecomRepoScore{}, database.RecomWeight{})
+ if err != nil {
+ return err
+ }
+
+ return initRecomWeights(db)
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.RecomOpWeight{}, database.RecomRepoScore{}, database.RecomWeight{})
+ })
+}
+
+func initRecomWeights(db *bun.DB) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ rw := &database.RecomWeight{
+ Name: "freshness",
+ WeightExp: expFreshness,
+ }
+ _, err := db.NewInsert().Model(rw).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to insert freshness recom weight")
+ }
+ rw = &database.RecomWeight{
+ Name: "downloads",
+ WeightExp: expDownloads,
+ }
+ _, err = db.NewInsert().Model(rw).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to insert downloads recom weight")
+ }
+ return nil
+}
+
+const expFreshness = `
+if hours <= 48{
+ score = 100 - 2.0/48.0 * hours
+} else if hours <= 168{
+ score = 98 - 38.0/120.0 * (hours - 48)
+} else{
+ score = 60 * (1 / (((hours - 168) / 48.0) + 1))
+}
+`
+
+const expDownloads = `
+if downloads <= 10{
+ score = downloads * (10 / 10)
+}else if downloads <= 100{
+ score = 10 + (downloads - 10) * (40.0 / 90.0)
+}else if downloads <= 1000{
+ score = 50 + (downloads - 100) * (40.0 / 900.0)
+}else{
+ score = 90 + 10 * (1 - (1 / (downloads / 1000.0)))
+}
+`
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, database.Mirror{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*database.Mirror)(nil)).
+ Index("idx_mirrors_repository_id").
+ Column("repository_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.Mirror{})
+ })
+
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.MirrorSource{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.MirrorSource{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, database.RuntimeFramework{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.RuntimeFramework{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type ClusterInfo struct {
+ ClusterID string `bun:",pk" json:"cluster_id"`
+ ClusterConfig string `bun:",notnull" json:"cluster_config"`
+ Region string `bun:",notnull" json:"region"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, ClusterInfo{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, ClusterInfo{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ if err := createTables(ctx, db, database.Event{}); err != nil {
+ return err
+ }
+
+ _, err := db.NewCreateIndex().Model(&database.Event{}).
+ Index("idx_events_created_at").
+ Column("created_at").
+ Exec(ctx)
+ return err
+
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.Event{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, database.RepositoriesRuntimeFramework{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.RepositoriesRuntimeFramework{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, database.SyncVersion{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.SyncVersion{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+)
+
+type AccountStatement struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ UserID string `bun:",notnull" json:"user_id"`
+ Value float64 `bun:",notnull" json:"value"`
+ Scene int `bun:",notnull" json:"scene"`
+ OpUID int64 `bun:",nullzero" json:"op_uid"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountStatement{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountStatement{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type AccountUser struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID string `bun:",notnull" json:"user_id"`
+ Balance float64 `bun:",notnull" json:"balance"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountUser{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountUser{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+)
+
+type AccountEvent struct {
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ EventBody map[string]string `bun:",hstore" json:"event_body"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountEvent{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountEvent{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type MirrorToken struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Token string `bun:",notnull" json:"token"`
+ ConcurrentCount int `bun:",nullzero" json:"concurrent_count"`
+ MaxBandwidth int `bun:",nullzero" json:"max_bandwidth"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, MirrorToken{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, MirrorToken{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+type AccountBill struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ BillDate time.Time `bun:"type:date" json:"bill_date"`
+ UserID string `bun:",notnull" json:"user_id"`
+ Scene int `bun:",notnull" json:"scene"`
+ CustomerID string `bun:",notnull" json:"customer_id"`
+ Value float64 `bun:",notnull" json:"value"`
+ Consumption float64 `bun:",notnull" json:"consumption"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountBill{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountBill{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type AccountSyncQuota struct {
+ UserID int64 `bun:",pk" json:"user_id"`
+ RepoCountLimit int64 `bun:",notnull" json:"repo_count_limit"`
+ RepoCountUsed int64 `bun:",notnull" json:"repo_count_used"`
+ SpeedLimit int64 `bun:",notnull" json:"speed_limit"`
+ TrafficLimit int64 `bun:",notnull" json:"traffic_limit"`
+ TrafficUsed int64 `bun:",notnull" json:"traffic_used"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountSyncQuota{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountSyncQuota{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+type AccountSyncQuotaStatement struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ RepoPath string `bun:",notnull" json:"repo_path"`
+ RepoType string `bun:",notnull" json:"repo_type"`
+ CreatedAt time.Time `bun:",notnull,default:current_timestamp" json:"created_at"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountSyncQuotaStatement{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountSyncQuotaStatement{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type File struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `json:"name"`
+ Path string `json:"path"`
+ ParentPath string `json:"parent_path"`
+ Size int64 `json:"size"`
+ LastCommitMessage string `json:"last_commit_message"`
+ LastCommitDate string `json:"last_commit_date"`
+ RepositoryID int64 `json:"repository_id"`
+ Repository *database.Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, File{})
+
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, File{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := db.NewSelect().
+ Model(&database.SyncClientSetting{}).
+ Scan(ctx)
+ if err == nil {
+ return nil
+ }
+
+ syncClientSetting := database.SyncClientSetting{
+ Token: "225caecb203219c972ae8d4368b93f868e6aed5a",
+ ConcurrentCount: 1,
+ MaxBandwidth: 8192000,
+ IsDefault: true,
+ }
+ return db.NewInsert().
+ Model(&syncClientSetting).
+ Scan(ctx)
+ }, func(ctx context.Context, db *bun.DB) error {
+ return nil
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type AccountPrice struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ SkuType int `bun:",notnull" json:"sku_type"`
+ SkuPrice int64 `bun:",notnull" json:"sku_price"`
+ SkuUnit int64 `bun:",notnull" json:"sku_unit"`
+ SkuDesc string `bun:",notnull" json:"sku_desc"`
+ ResourceID string `bun:",notnull" json:"resource_id"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountPrice{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountPrice{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+)
+
+type AccountMetering struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ Value float64 `bun:",notnull" json:"value"`
+ ValueType int `bun:",notnull" json:"value_type"`
+ Scene int `bun:",notnull" json:"scene"`
+ OpUID string `json:"op_uid"`
+ ResourceID string `bun:",notnull" json:"resource_id"`
+ ResourceName string `bun:",notnull" json:"resource_name"`
+ CustomerID string `json:"customer_id"`
+ RecordedAt time.Time `bun:",notnull" json:"recorded_at"`
+ Extra string `json:"extra"`
+ CreatedAt time.Time `bun:",notnull,default:current_timestamp" json:"created_at"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountMetering{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountMetering{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type LfsMetaObject struct {
+ ID int64 `bun:",pk,autoincrement" json:"user_id"`
+ Oid string `bun:",notnull" json:"oid"`
+ Size int64 `bun:",notnull" json:"size"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository database.Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ Existing bool `bun:",notnull" json:"existing"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, LfsMetaObject{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, LfsMetaObject{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+var collectionTables = []any{
+ database.CollectionRepository{},
+ database.Collection{},
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, collectionTables...)
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, collectionTables...)
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type LfsLock struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository database.Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User database.User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ Path string `bun:",notnull" json:"path"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, LfsLock{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ fmt.Print(" [down migration] ")
+ return dropTables(ctx, db, LfsLock{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return createTables(ctx, db, Telemetry{})
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, Telemetry{})
+ })
+}
+
+type Telemetry struct {
+ UUID string `bun:"" json:"uuid"`
+ RecordedAt time.Time `bun:"" json:"recorded_at"`
+ Hostname string `bun:"" json:"hostname,omitempty"`
+ Version string `bun:"" json:"version"`
+ InstallationType string `bun:"" json:"installation_type,omitempty"`
+ ActiveUserCount int `bun:"" json:"active_user_count"`
+ Edition string `bun:"" json:"edition,omitempty"`
+ LicenseMD5 string `bun:"" json:"license_md5,omitempty"`
+ LicenseID int `bun:"" json:"license_id,omitempty"`
+ HistoricalMaxUsers int `bun:"" json:"historical_max_users,omitempty"`
+ Licensee interface{} `bun:"type:jsonb" json:"licensee,omitempty"`
+ LicenseUserCount int `bun:"" json:"license_user_count,omitempty"`
+ LicenseBillableUsers int `bun:"" json:"license_billable_users,omitempty"`
+ LicenseStartsAt time.Time `bun:"" json:"license_starts_at,omitempty"`
+ LicenseExpiresAt time.Time `bun:"" json:"license_expires_at,omitempty"`
+ LicensePlan string `bun:"" json:"license_plan,omitempty"`
+ LicenseAddOns map[string]interface{} `bun:"type:jsonb" json:"license_add_ons,omitempty"`
+ Settings interface{} `bun:"type:jsonb" json:"settings,omitempty"`
+ Counts interface{} `bun:"type:jsonb" json:"counts,omitempty"`
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type RuntimeArchitecture struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RuntimeFrameworkID int64 `bun:",notnull" json:"runtime_framework_id"`
+ ArchitectureName string `bun:",notnull" json:"architecture_name"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, RuntimeArchitecture{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, RuntimeArchitecture{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, database.RepositoryFile{}, RepositoryFileCheck{})
+ if err != nil {
+ return fmt.Errorf("create table repository_files: %w", err)
+ }
+
+ _, err = db.NewCreateIndex().
+ Model((*RepositoryFile)(nil)).
+ Index("repository_files_path_idx").
+ Column("repository_id", "branch").
+ IfNotExists().
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("create index repository_files_path_idx: %w", err)
+ }
+
+ _, err = db.NewCreateIndex().
+ Model((*RepositoryFileCheck)(nil)).
+ Index("repository_file_checks_repo_file_id_idx").
+ Column("repo_file_id").
+ Unique().
+ IfNotExists().
+ Exec(ctx)
+
+ if err != nil {
+ return fmt.Errorf("create index repository_file_checks_repo_file_id_idx: %w", err)
+ }
+
+ return nil
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.RepositoryFile{}, RepositoryFileCheck{})
+ })
+
+}
+
+type RepositoryFile struct {
+ ID int64 `bun:",pk,autoincrement" `
+ RepositoryID int64 `bun:",notnull" `
+ Path string `bun:",notnull" `
+ FileType string `bun:",notnull" `
+ Size int64 `bun:",nullzero" `
+ LastModify time.Time `bun:",nullzero" `
+ CommitSha string `bun:",nullzero" `
+ LfsRelativePath string `bun:",nullzero" `
+ Branch string `bun:",nullzero" `
+ Repository *database.Repository `bun:"rel:belongs-to,join:repository_id=id"`
+}
+
+type RepositoryFileCheck struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepoFileID int64 `bun:"," json:"repo_file_id"`
+ Status types.SensitiveCheckStatus `bun:",notnull" json:"status"`
+ Message string `bun:",nullzero" json:"message"`
+ CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
+ //uuid for async check task
+ TaskID string `bun:",nullzero" json:"task_id"`
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/google/uuid"
+ "github.com/uptrace/bun"
+)
+
+type AccountPresent struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ EventUUID uuid.UUID `bun:"type:uuid,notnull" json:"event_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ ActivityID int64 `bun:",notnull" json:"activity_id"`
+ Value float64 `bun:",notnull" json:"value"`
+ OpUID string `bun:",notnull" json:"op_uid"`
+ OpDesc string `bun:",notnull" json:"op_desc"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountPresent{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*AccountPresent)(nil)).
+ Index("idx_account_present_useruuid_activityid").
+ Column("user_uuid", "activity_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountPresent{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type AccountOrder struct {
+ OrderUUID string `bun:",notnull,pk" json:"order_uuid"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ OrderStatus types.OrderStatus `bun:",notnull" json:"order_status"`
+ Amount float64 `bun:",notnull" json:"amount"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ EventUUID string `json:"event_uuid"`
+ RecordedAt time.Time `json:"recorded_at"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountOrder{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*AccountOrder)(nil)).
+ Index("idx_account_order_createat_useruuid_status").
+ Column("created_at", "user_uuid", "order_status").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountOrder{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type AccountOrderDetail struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ OrderUUID string `bun:",notnull" json:"order_uuid"`
+ ResourceID string `bun:",notnull" json:"resource_id"`
+ SkuType types.SKUType `bun:",notnull" json:"sku_type"`
+ SkuKind types.SKUKind `bun:",notnull" json:"sku_kind"`
+ SkuUnitType string `bun:",notnull" json:"sku_unit_type"`
+ OrderCount int `bun:",notnull" json:"order_count"`
+ SkuPriceID int64 `bun:",notnull" json:"sku_price_id"`
+ Amount float64 `bun:",notnull" json:"amount"`
+ BeginTime time.Time `bun:",notnull" json:"begin_time"`
+ EndTime time.Time `bun:",notnull" json:"end_time"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ PresentUUID string `json:"present_uuid"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountOrderDetail{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*AccountOrderDetail)(nil)).
+ Index("idx_account_order_detail_orderuuid").
+ Column("order_uuid").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, AccountOrderDetail{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type UserResources struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserUID string `bun:",notnull" json:"user_uid"`
+ OrderId string `bun:",notnull" json:"order_id"`
+ OrderDetailId int64 `bun:",notnull,unique" json:"order_detail_id"`
+ ResourceId int64 `bun:",notnull" json:"resource_id"`
+ DeployId int64 `bun:",notnull" json:"deploy_id"`
+ XPUNum int `bun:",notnull" json:"xpu_num"`
+ PayMode string `bun:",notnull" json:"pay_mode"`
+ Price float64 `bun:",notnull" json:"price"`
+ CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ StartTime time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"start_time"`
+ EndTime time.Time `bun:",nullzero,notnull,skipupdate" json:"end_time"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, UserResources{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*database.UserResources)(nil)).
+ Index("idx_user_resources_useruid").
+ Column("user_uid", "order_detail_id", "end_time").
+ Exec(ctx)
+ return err
+
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.UserResources{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "strings"
+
+ "github.com/uptrace/bun"
+ "golang.org/x/crypto/ssh"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ return calculateSSHKeyFingerprint(ctx, db)
+ }, func(ctx context.Context, db *bun.DB) error {
+ return nil
+ })
+}
+
+func calculateSSHKeyFingerprint(ctx context.Context, db *bun.DB) error {
+ var sshKeys []database.SSHKey
+ err := db.NewSelect().
+ Model(&database.SSHKey{}).
+ Where("fingerprint_sha256 is NULL").
+ Scan(ctx, &sshKeys)
+ if err != nil {
+ return err
+ }
+
+ for _, sshKey := range sshKeys {
+ parsedKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(sshKey.Content))
+ if err != nil {
+ return err
+ }
+ fingerPrint := ssh.FingerprintSHA256(parsedKey)
+ fingerPrint = strings.Split(fingerPrint, ":")[1]
+ sshKey.FingerprintSHA256 = fingerPrint
+ _, err = db.NewUpdate().
+ Model(&sshKey).
+ WherePK().
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ }
+
+ return err
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type PromptPrefix struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ZH string `bun:",notnull" json:"zh"`
+ EN string `bun:",notnull" json:"en"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, PromptPrefix{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, PromptPrefix{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type PromptConversation struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ ConversationID string `bun:",notnull" json:"conversation_id"`
+ Title string `bun:",notnull" json:"title"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, PromptConversation{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*PromptConversation)(nil)).
+ Index("idx_unique_prompt_conversation_conversationid").
+ Column("conversation_id").
+ Unique().
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*PromptConversation)(nil)).
+ Index("idx_prompt_conversation_userid").
+ Column("user_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, PromptConversation{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type PromptConversationMessage struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ConversationID string `bun:",notnull" json:"conversation_id"`
+ Role string `bun:",notnull" json:"role"`
+ Content string `bun:",notnull" json:"content"`
+ UserLike bool `bun:",notnull" json:"user_like"`
+ UserHate bool `bun:",notnull" json:"user_hate"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, PromptConversationMessage{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*PromptConversationMessage)(nil)).
+ Index("idx_prompt_conversation_message_conversationid").
+ Column("conversation_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, PromptConversationMessage{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, Discussion{}, Comment{})
+ if err != nil {
+ return err
+ }
+
+ //create index for table discussions
+ _, err = db.NewCreateIndex().Model(&Discussion{}).
+ Column("discussionable_type", "discussionable_id").
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to create index for table discussion: %w", err)
+ }
+ //create index for table comments
+ _, err = db.NewCreateIndex().Model(&Comment{}).
+ Column("commentable_type", "commentable_id").
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to create index for table comment: %w", err)
+ }
+ return nil
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, Discussion{}, Comment{})
+ })
+}
+
+type Discussion struct {
+ ID int64 `bun:"id,pk,autoincrement"`
+ UserID int64 `bun:"user_id,notnull"`
+ Title string `bun:"title,notnull"`
+ DiscussionableID int64 `bun:"discussionable_id,notnull"`
+ DiscussionableType string `bun:"discussionable_type,notnull"`
+ CommentCount int64 `bun:"comment_count,notnull,default:0"`
+ CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp"`
+ UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp"`
+}
+
+type Comment struct {
+ ID int64 `bun:"id,pk,autoincrement"`
+ Content string `bun:"content"`
+ CommentableType string `bun:"commentable_type,notnull"`
+ CommentableID int64 `bun:"commentable_id,notnull"`
+ UserID int64 `bun:"user_id,notnull"`
+ CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp"`
+ UpdatedAt time.Time `bun:"updated_at,notnull,default:current_timestamp"`
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type LLMConfig struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ModelName string `bun:",notnull" json:"model_name"`
+ ApiEndpoint string `bun:",notnull" json:"api_endpoint"`
+ AuthHeader string `bun:",notnull" json:"auth_header"`
+ Type int `bun:",notnull" json:"type"`
+ Enabled bool `bun:",notnull" json:"enabled"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, LLMConfig{})
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, LLMConfig{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, ResourceModel{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*ResourceModel)(nil)).
+ Index("idx_resource_model_name").
+ Column("resource_name", "engine_name", "model_name").
+ Exec(ctx)
+ return err
+
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, ResourceModel{})
+ })
+}
+
+type ResourceModel struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ResourceName string `bun:",notnull" json:"resource_name"`
+ EngineName string `bun:",notnull" json:"engine_name"`
+ ModelName string `bun:",notnull" json:"model_name"`
+ Type string `bun:",notnull" json:"type"`
+ times
+}
+
+
+
package migrations
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type Prompt struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, Prompt{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*Prompt)(nil)).
+ Index("idx_prompts_repositoryid").
+ Column("repository_id").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, Prompt{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type ArgoWorkflow struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Username string `bun:",notnull" json:"username"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ TaskName string `bun:",notnull" json:"task_name"` // user input name
+ TaskId string `bun:",notnull" json:"task_id"` // generated task id
+ TaskType types.TaskType `bun:",notnull" json:"task_type"`
+ RepoIds []string `bun:",notnull,type:jsonb" json:"repo_ids"`
+ RepoType string `bun:",notnull" json:"repo_type"`
+ TaskDesc string `bun:"," json:"task_desc"`
+ Status v1alpha1.WorkflowPhase `bun:"," json:"status"`
+ Reason string `bun:"," json:"reason"` // reason for status
+ Image string `bun:",notnull" json:"image"` // ArgoWorkFlow framework
+ Datasets []string `bun:",notnull,type:jsonb" json:"datasets"`
+ ResourceId int64 `bun:",nullzero" json:"resource_id"`
+ SubmitTime time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"submit_time"`
+ StartTime time.Time `bun:",nullzero" json:"start_time"`
+ EndTime time.Time `bun:",nullzero" json:"end_time"`
+ ResultURL string `bun:",nullzero" json:"result_url"`
+ DownloadURL string `bun:",nullzero" json:"download_url"`
+ FailuresURL string `bun:",nullzero" json:"failures_url"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, ArgoWorkflow{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*ArgoWorkflow)(nil)).
+ Index("idx_workflow_user_uuid").
+ Column("username", "task_id").
+ Exec(ctx)
+ return err
+
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, ArgoWorkflow{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+type License struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Key string `bun:",notnull" json:"key"`
+ Company string `bun:",notnull" json:"company"`
+ Email string `bun:",notnull" json:"email"`
+ Product string `bun:",notnull" json:"product"`
+ Edition string `bun:",notnull" json:"edition"`
+ Version string `bun:",nullzero" json:"version"`
+ Status string `bun:",nullzero" json:"status"`
+ MaxUser int `bun:",notnull" json:"max_user"`
+ StartTime time.Time `bun:",notnull" json:"start_time"`
+ ExpireTime time.Time `bun:",notnull" json:"expire_time"`
+ Extra string `bun:",nullzero" json:"extra"`
+ Remark string `bun:",nullzero" json:"remark"`
+ UserUUID string `bun:",notnull" json:"user_uuid"`
+ times
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, License{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*License)(nil)).
+ Index("idx_license_product_edition").
+ Column("product", "edition").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, License{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "time"
+
+ "github.com/uptrace/bun"
+)
+
+type TagRule struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepoName string `bun:",notnull" json:"repo_name"`
+ RepoType string `bun:",notnull" json:"repo_type"`
+ Category string `bun:",notnull" json:"category"`
+ TagName string `bun:",notnull" json:"tag_name"`
+ Tag Tag `bun:",rel:has-one,join:tag_name=name"`
+ CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, TagRule{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*TagRule)(nil)).
+ Index("idx_dataset_tag_name_type").
+ Column("repo_name", "repo_type").
+ Exec(ctx)
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, TagRule{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "time"
+)
+
+type Payment struct {
+ bun.BaseModel `bun:"table:payment_payment"`
+
+ PaymentUUID string `bun:",notnull,pk,skipupdate" json:"payment_uuid"`
+
+ // Transaction serial number returned by the payment channel.
+ TransactionNo string `json:"transaction_no"`
+
+ // Order number, tailored to the requirements of each channel, and must be unique within the business system.
+ // For example, in the case of a recharge, this field corresponds to the orderNo in the recharge table.
+ // For payment channels, this parameter typically corresponds to out_trade_no.
+ OrderNo string `bun:",notnull,skipupdate" json:"order_no"`
+
+ // Payment channel.
+ Channel consts.PaymentChannel `bun:",notnull,skipupdate" json:"channel"`
+
+ // Transformed into a QR code for frontend scanning payment scenarios.
+ CodeUrl string `bun:",skipupdate" json:"code_url"`
+
+ // Payment credentials used by the client to initiate a payment.
+ Credentials json.RawMessage `bun:",nullzero" json:"credentials"`
+
+ // Client IP address.
+ ClientIp string `bun:",skipupdate" json:"client_ip"`
+
+ // Total amount in the smallest currency unit (e.g., in CNY, this is expressed in cents).
+ Amount float64 `bun:",notnull,skipupdate" json:"amount"`
+
+ // 3-letter ISO currency code, represented in uppercase letters.
+ Currency string `bun:",notnull,skipupdate,default:'CNY'" json:"currency"`
+
+ // Product title, limited to a maximum of 32 Unicode characters.
+ Subject string `bun:",notnull,skipupdate" json:"subject"`
+
+ // Product description, limited to a maximum of 128 Unicode characters.
+ // Note: yeepay_wap restricts this parameter to a maximum of 100 Unicode characters;
+ // some channels of Alipay do not support special characters.
+ Body string `bun:",skipupdate" json:"body"`
+
+ // Custom fields for business-specific use cases.
+ Extra string `bun:",skipupdate" json:"extra"`
+
+ // Indicates whether the payment has been completed.
+ Paid bool `json:"paid"`
+
+ // Indicates whether the order has been revoked.
+ Reversed bool `json:"reversed"`
+
+ // Unix timestamp representing the time when the payment was completed.
+ TimePaid time.Time `bun:",nullzero" json:"time_paid"`
+
+ // Unix timestamp representing the expiration time of the order.
+ TimeExpire time.Time `bun:",nullzero" json:"time_expire"`
+
+ // Payment creation time.
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+
+ // Payment update time.
+ UpdatedAt time.Time `bun:",notnull,default:current_timestamp" json:"updated_at"`
+
+ // Error code returned in case of payment failure.
+ FailureCode string `json:"failure_code"`
+
+ // Error message or description for the payment failure.
+ FailureMsg string `json:"failure_msg"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, Payment{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*Payment)(nil)).
+ Index("idx_payment_order_no").
+ Column("order_no").
+ Unique().
+ IfNotExists().
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("create index idx_payment_order_no fail: %w", err)
+ }
+ return err
+
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, Payment{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "fmt"
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+ "time"
+)
+
+type AccountRecharge struct {
+ RechargeUUID string `bun:",notnull,pk,skipupdate" json:"uuid"` // Recharge object ID
+ OrderNo string `bun:",notnull,unique" json:"order_no"` // Order ID allowed by the payment system
+ UserUUID string `bun:",notnull,skipupdate" json:"user_uuid"` // Target UserUUID for the recharge
+ FromUserUUID string `bun:",notnull,skipupdate" json:"from_user_uuid"` // Source UserUUID for the recharge
+ Amount float64 `bun:",notnull,skipupdate" json:"amount"` // Actual balance received by the user, in cents
+ Currency string `bun:",notnull,skipupdate,default:'CNY'" json:"currency"` // 3-letter ISO currency code in uppercase letters
+ Payment *Payment `bun:"rel:belongs-to,join:payment_uuid=payment_uuid" json:"payment"`
+ PaymentUUID string `bun:",notnull,skipupdate,unique" json:"payment_uuid"`
+ Succeeded bool `json:"succeeded"`
+ Closed bool `json:"closed"`
+ TimeSucceeded time.Time `bun:",nullzero" json:"time_succeeded"`
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ UpdatedAt time.Time `bun:",notnull,default:current_timestamp" json:"updated_at"`
+ Description string `json:"description"`
+}
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ err := createTables(ctx, db, AccountRecharge{})
+ if err != nil {
+ return err
+ }
+ _, err = db.NewCreateIndex().
+ Model((*AccountRecharge)(nil)).
+ Index("idx_payment_uuid").
+ Column("payment_uuid").
+ Unique().
+ IfNotExists().
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("create index idx_payment_uuid fail: %w", err)
+ }
+ return err
+ }, func(ctx context.Context, db *bun.DB) error {
+ return dropTables(ctx, db, database.AccountRecharge{})
+ })
+}
+
+
+
package migrations
+
+import (
+ "context"
+ "fmt"
+ "github.com/uptrace/bun"
+)
+
+func init() {
+ Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
+ // Up migration: Change 'amount' from float64 to int64
+
+ // Adjust table names if necessary
+ paymentTableName := "payment_payment"
+ accountRechargeTableName := "account_recharges" // Use "account_recharge" if that's the correct name
+
+ // Alter 'amount' column in 'payment_payment' table
+ _, err := db.ExecContext(ctx, fmt.Sprintf(`
+ ALTER TABLE %s
+ ALTER COLUMN amount TYPE bigint USING ROUND(amount)::bigint;
+ `, paymentTableName))
+ if err != nil {
+ return fmt.Errorf("failed to alter 'amount' column in 'payment_payment' table: %w", err)
+ }
+
+ // Alter 'amount' column in 'account_recharge' table
+ _, err = db.ExecContext(ctx, fmt.Sprintf(`
+ ALTER TABLE %s
+ ALTER COLUMN amount TYPE bigint USING ROUND(amount)::bigint;
+ `, accountRechargeTableName))
+ if err != nil {
+ return fmt.Errorf("failed to alter 'amount' column in 'account_recharge' table: %w", err)
+ }
+
+ return nil
+ }, func(ctx context.Context, db *bun.DB) error {
+ // Down migration: Revert 'amount' back to float64
+
+ paymentTableName := "payment_payment"
+ accountRechargeTableName := "account_recharges" // Use "account_recharge" if that's the correct name
+
+ // Revert 'amount' column in 'payment_payment' table
+ _, err := db.ExecContext(ctx, fmt.Sprintf(`
+ ALTER TABLE %s
+ ALTER COLUMN amount TYPE double precision USING amount::double precision;
+ `, paymentTableName))
+ if err != nil {
+ return fmt.Errorf("failed to revert 'amount' column in 'payment_payment' table: %w", err)
+ }
+
+ // Revert 'amount' column in 'account_recharge' table
+ _, err = db.ExecContext(ctx, fmt.Sprintf(`
+ ALTER TABLE %s
+ ALTER COLUMN amount TYPE double precision USING amount::double precision;
+ `, accountRechargeTableName))
+ if err != nil {
+ return fmt.Errorf("failed to revert 'amount' column in 'account_recharge' table: %w", err)
+ }
+
+ return nil
+ })
+}
+
+
+
// Package migrations supports database migration.
+// Mind it can not depend on model package,
+// as it's the latter's dependency.
+package migrations
+
+import (
+ "context"
+ "database/sql"
+ "embed"
+ "fmt"
+ "reflect"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/builder/store/database"
+
+ "github.com/uptrace/bun/migrate"
+)
+
+//go:embed *.sql
+var migrationDir embed.FS
+
+var Migrations = migrate.NewMigrations()
+
+func init() {
+ if err := Migrations.Discover(migrationDir); err != nil {
+ err = fmt.Errorf("discovering database migrations: %w", err)
+ panic(err)
+ }
+}
+
+// NewMigrator factory of database migrator
+func NewMigrator(db *database.DB) *migrate.Migrator {
+ return migrate.NewMigrator(db.BunDB, Migrations) // nolint: staticcheck
+}
+
+func createTables(ctx context.Context, db *bun.DB, tables ...any) (err error) {
+ err = db.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, tx bun.Tx) (err error) {
+ for _, table := range tables {
+ tableType := reflect.TypeOf(table)
+ if tableType.Kind() == reflect.Pointer {
+ tableType = tableType.Elem()
+ }
+ if tableType.Kind() != reflect.Struct {
+ err = fmt.Errorf("input %T is of kind %s, want struct", table, tableType.Kind().String())
+ return
+ }
+
+ _, err = tx.NewCreateTable().
+ Model(reflect.New(tableType).Interface()).
+ Exec(ctx)
+ if err != nil {
+ err = fmt.Errorf("creating table for %T: %w", table, err)
+ return
+ }
+ }
+
+ return
+ })
+
+ return
+}
+
+func dropTables(ctx context.Context, db *bun.DB, tables ...any) (err error) {
+ err = db.RunInTx(ctx, &sql.TxOptions{}, func(ctx context.Context, tx bun.Tx) (err error) {
+ for _, table := range tables {
+ tableType := reflect.TypeOf(table)
+ if tableType.Kind() == reflect.Pointer {
+ tableType = tableType.Elem()
+ }
+ if tableType.Kind() != reflect.Struct {
+ err = fmt.Errorf("input %T is of kind %s, want struct", table, tableType.Kind().String())
+ return
+ }
+
+ _, err = tx.NewDropTable().
+ Model(reflect.New(tableType).Interface()).
+ IfExists().
+ Cascade().
+ Exec(ctx)
+ if err != nil {
+ err = fmt.Errorf("creating table for %T: %w", table, err)
+ return
+ }
+ }
+
+ return
+ })
+
+ return
+}
+
+// times has now as its default on inserting.
+// Updating operations must take care of UpdatedAt by themselves.
+// CreatedAt will not be updated thanks to skipupdate field tag,
+// ref: https://github.com/uptrace/bun/pull/565
+// Sadly, bun.BeforeAppendModelHook does not actually work,
+// see https://github.com/uptrace/bun/discussions/621
+type times struct {
+ CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp" json:"updated_at"`
+}
+
+// OwnerType is the owner a user or organization?
+type OwnerType string
+
+// OwnerRef tracks resource owner from User or Organization
+type OwnerRef struct {
+ // is the owner a user or organization?
+ OwnerType OwnerType `bun:",nullzero,notnull" json:"ownerType"`
+ // foreigner key to User / Organization
+ OwnerID int `bun:",nullzero,notnull" json:"ownerId"`
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type mirrorStoreImpl struct {
+ db *DB
+}
+
+type MirrorStore interface {
+ IsExist(ctx context.Context, repoID int64) (exists bool, err error)
+ IsRepoExist(ctx context.Context, repoType types.RepositoryType, namespace, name string) (exists bool, err error)
+ FindByRepoID(ctx context.Context, repoID int64) (*Mirror, error)
+ FindByID(ctx context.Context, ID int64) (*Mirror, error)
+ FindByRepoPath(ctx context.Context, repoType types.RepositoryType, namespace, name string) (*Mirror, error)
+ FindWithMapping(ctx context.Context, repoType types.RepositoryType, namespace, name string, mapping types.Mapping) (*Mirror, error)
+ Create(ctx context.Context, mirror *Mirror) (*Mirror, error)
+ WithPagination(ctx context.Context) ([]Mirror, error)
+ WithPaginationWithRepository(ctx context.Context) ([]Mirror, error)
+ NoPushMirror(ctx context.Context) ([]Mirror, error)
+ PushedMirror(ctx context.Context) ([]Mirror, error)
+ Update(ctx context.Context, mirror *Mirror) (err error)
+ Delete(ctx context.Context, mirror *Mirror) (err error)
+ Unfinished(ctx context.Context) ([]Mirror, error)
+ Finished(ctx context.Context) ([]Mirror, error)
+ ToSyncRepo(ctx context.Context) ([]Mirror, error)
+ ToSyncLfs(ctx context.Context) ([]Mirror, error)
+ IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error)
+ StatusCount(ctx context.Context) ([]MirrorStatusCount, error)
+ UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error
+}
+
+func NewMirrorStore() MirrorStore {
+ return &mirrorStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewMirrorStoreWithDB(db *DB) MirrorStore {
+ return &mirrorStoreImpl{
+ db: db,
+ }
+}
+
+type Mirror struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Interval string `bun:",notnull" json:"interval"`
+ SourceUrl string `bun:",notnull" json:"source_url"`
+ MirrorSourceID int64 `bun:",notnull" json:"mirror_source_id"`
+ MirrorSource MirrorSource `bun:"rel:belongs-to,join:mirror_source_id=id" json:"mirror_source"`
+ //source user name
+ Username string `bun:",nullzero" json:"-"`
+ // source access token
+ AccessToken string `bun:",nullzero" json:"-"`
+ PushUrl string `bun:",nullzero" json:"-"`
+ PushUsername string `bun:",nullzero" json:"-"`
+ PushAccessToken string `bun:",nullzero" json:"-"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ LastUpdatedAt time.Time `bun:",nullzero" json:"last_updated_at"`
+ SourceRepoPath string `bun:",nullzero" json:"source_repo_path"`
+ LocalRepoPath string `bun:",nullzero" json:"local_repo_path"`
+ LastMessage string `bun:",nullzero" json:"last_message"`
+ MirrorTaskID int64 `bun:",nullzero" json:"mirror_task_id"`
+ PushMirrorCreated bool `bun:",nullzero,default:false" json:"push_mirror_created"`
+ Status types.MirrorTaskStatus `bun:",nullzero" json:"status"`
+ Progress int8 `bun:",nullzero" json:"progress"`
+ NextExecutionTimestamp time.Time `bun:",nullzero" json:"next_execution_timestamp"`
+ Priority types.MirrorPriority `bun:"mirror_priority,notnull,default:0" json:"priority"`
+
+ times
+}
+
+type MirrorStatusCount struct {
+ Status types.MirrorTaskStatus `bun:"status"`
+ Count int `bun:"count"`
+}
+
+func (s *mirrorStoreImpl) IsExist(ctx context.Context, repoID int64) (exists bool, err error) {
+ var mirror Mirror
+ exists, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&mirror).
+ Where("repository_id=?", repoID).
+ Exists(ctx)
+ return
+}
+func (s *mirrorStoreImpl) IsRepoExist(ctx context.Context, repoType types.RepositoryType, namespace, name string) (exists bool, err error) {
+ var repo Repository
+ exists, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&repo).
+ Where("path=?", fmt.Sprintf("%s/%s", namespace, name)).
+ Where("repository_type=?", repoType).
+ Exists(ctx)
+ return
+}
+
+func (s *mirrorStoreImpl) FindByRepoID(ctx context.Context, repoID int64) (*Mirror, error) {
+ var mirror Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirror).
+ Where("repository_id=?", repoID).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &mirror, nil
+}
+
+func (s *mirrorStoreImpl) FindByID(ctx context.Context, ID int64) (*Mirror, error) {
+ var mirror Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirror).
+ Relation("Repository").
+ Where("mirror.id=?", ID).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &mirror, nil
+}
+
+func (s *mirrorStoreImpl) FindByRepoPath(ctx context.Context, repoType types.RepositoryType, namespace, name string) (*Mirror, error) {
+ var mirror Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirror).
+ Join("JOIN repositories AS r ON mirror.repository_id = r.id ").
+ Where("LOWER(r.git_path) = LOWER(?)", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &mirror, nil
+}
+
+func (s *mirrorStoreImpl) FindWithMapping(ctx context.Context, repoType types.RepositoryType, namespace, name string, mapping types.Mapping) (*Mirror, error) {
+ var mirror Mirror
+ var err error
+ if mapping == types.CSGHubMapping {
+ return s.FindByRepoPath(ctx, repoType, namespace, name)
+ } else if mapping == types.HFMapping {
+ err = s.db.Operator.Core.NewSelect().
+ Model(&mirror).
+ Relation("Repository").
+ Where("mirror.source_repo_path=?", fmt.Sprintf("%s/%s", namespace, name)).
+ Where("repository.repository_type=?", repoType).
+ Scan(ctx)
+ } else {
+ // auto mapping
+ //fix some repo id has mirror but it's not public,for example: https://opencsg.com/models/Qwen/Qwen_Qwen2-7B-Instruct
+ exist, _ := s.IsRepoExist(ctx, repoType, namespace, name)
+ if exist {
+ // no need mapping if repo id already exists in reporitory
+ return nil, fmt.Errorf("repo already exists, no need mapping")
+ }
+ err = s.db.Operator.Core.NewSelect().
+ Model(&mirror).
+ Relation("Repository").
+ Where("mirror.source_repo_path=?", fmt.Sprintf("%s/%s", namespace, name)).
+ Where("repository.repository_type=?", repoType).
+ Scan(ctx)
+ }
+ if err != nil {
+ return nil, err
+ }
+ return &mirror, nil
+}
+
+func (s *mirrorStoreImpl) Create(ctx context.Context, mirror *Mirror) (*Mirror, error) {
+ err := s.db.Operator.Core.NewInsert().
+ Model(mirror).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirror, nil
+}
+
+func (s *mirrorStoreImpl) WithPagination(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) WithPaginationWithRepository(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Relation("Repository").
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) NoPushMirror(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Where("push_mirror_created = ?", false).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) PushedMirror(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Relation("Repository").
+ Where("push_mirror_created = ?", true).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) Update(ctx context.Context, mirror *Mirror) (err error) {
+ err = assertAffectedOneRow(s.db.Operator.Core.NewUpdate().
+ Model(mirror).
+ WherePK().
+ Exec(ctx),
+ )
+
+ return
+}
+
+func (s *mirrorStoreImpl) Delete(ctx context.Context, mirror *Mirror) (err error) {
+ _, err = s.db.Operator.Core.
+ NewDelete().
+ Model(mirror).
+ WherePK().
+ Exec(ctx)
+ return
+}
+
+func (s *mirrorStoreImpl) Unfinished(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Relation("Repository").
+ Where("status != ? OR status IS NULL", types.MirrorFinished).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) Finished(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Relation("Repository").
+ Where("status = ?", types.MirrorFinished).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) ToSyncRepo(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Where(
+ "next_execution_timestamp < ? or status in (?,?,?,?)",
+ time.Now(),
+ types.MirrorIncomplete,
+ types.MirrorFailed,
+ types.MirrorWaiting,
+ types.MirrorRunning).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) ToSyncLfs(ctx context.Context) ([]Mirror, error) {
+ var mirrors []Mirror
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Where("next_execution_timestamp < ? or status = ?", time.Now(), types.MirrorRepoSynced).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrors, nil
+}
+
+func (s *mirrorStoreImpl) IndexWithPagination(ctx context.Context, per, page int, search string) (mirrors []Mirror, count int, err error) {
+ q := s.db.Operator.Core.NewSelect().
+ Model(&mirrors).
+ Relation("Repository").
+ Relation("MirrorSource")
+ if search != "" {
+ q = q.Where("LOWER(mirror.source_url) like ? or LOWER(mirror.local_repo_path) like ?",
+ fmt.Sprintf("%%%s%%", strings.ToLower(search)),
+ fmt.Sprintf("%%%s%%", strings.ToLower(search)),
+ )
+ }
+ count, err = q.Count(ctx)
+ if err != nil {
+ return
+ }
+ err = q.Limit(per).
+ Offset((page - 1) * per).
+ Scan(ctx)
+
+ if err != nil {
+ return
+ }
+
+ return
+}
+
+func (s *mirrorStoreImpl) StatusCount(ctx context.Context) ([]MirrorStatusCount, error) {
+ var statusCounts []MirrorStatusCount
+ err := s.db.Operator.Core.NewSelect().
+ Model((*Mirror)(nil)).
+ Column("status").
+ ColumnExpr("COUNT(*) AS count").
+ Group("status").
+ Scan(ctx, &statusCounts)
+ return statusCounts, err
+}
+
+func (s *mirrorStoreImpl) UpdateMirrorAndRepository(ctx context.Context, mirror *Mirror, repo *Repository) error {
+ err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ _, err := tx.NewUpdate().Model(mirror).WherePK().Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror: %v", err)
+ }
+ _, err = tx.NewUpdate().Model(repo).WherePK().Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to update repository: %v", err)
+ }
+ return nil
+ })
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "strings"
+)
+
+type mirrorSourceStoreImpl struct {
+ db *DB
+}
+
+type MirrorSourceStore interface {
+ Create(ctx context.Context, mirrorSource *MirrorSource) (*MirrorSource, error)
+ Index(ctx context.Context) ([]MirrorSource, error)
+ Get(ctx context.Context, id int64) (*MirrorSource, error)
+ FindByName(ctx context.Context, name string) (*MirrorSource, error)
+ Update(ctx context.Context, mirrorSource *MirrorSource) (err error)
+ Delete(ctx context.Context, mirrorSource *MirrorSource) (err error)
+}
+
+func NewMirrorSourceStore() MirrorSourceStore {
+ return &mirrorSourceStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewMirrorSourceStoreWithDB(db *DB) MirrorSourceStore {
+ return &mirrorSourceStoreImpl{
+ db: db,
+ }
+}
+
+type MirrorSource struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ SourceName string `bun:",notnull,unique" json:"source_name"`
+ InfoAPIUrl string `bun:",nullzero" json:"info_api_url"`
+
+ times
+}
+
+func (s *mirrorSourceStoreImpl) Create(ctx context.Context, mirrorSource *MirrorSource) (*MirrorSource, error) {
+ err := s.db.Operator.Core.NewInsert().
+ Model(mirrorSource).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrorSource, nil
+}
+
+func (s *mirrorSourceStoreImpl) Index(ctx context.Context) ([]MirrorSource, error) {
+ var mirrorSources []MirrorSource
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrorSources).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return mirrorSources, nil
+}
+
+func (s *mirrorSourceStoreImpl) Get(ctx context.Context, id int64) (*MirrorSource, error) {
+ var mirrorSource MirrorSource
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrorSource).
+ Where("id=?", id).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &mirrorSource, nil
+}
+
+func (s *mirrorSourceStoreImpl) FindByName(ctx context.Context, name string) (*MirrorSource, error) {
+ var mirrorSource MirrorSource
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mirrorSource).
+ Where("source_name=?", name).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &mirrorSource, nil
+}
+
+func (s *mirrorSourceStoreImpl) Update(ctx context.Context, mirrorSource *MirrorSource) (err error) {
+ err = assertAffectedOneRow(s.db.Operator.Core.NewUpdate().
+ Model(mirrorSource).
+ WherePK().
+ Exec(ctx),
+ )
+
+ return
+}
+
+func (s *mirrorSourceStoreImpl) Delete(ctx context.Context, mirrorSource *MirrorSource) (err error) {
+ _, err = s.db.Operator.Core.
+ NewDelete().
+ Model(mirrorSource).
+ WherePK().
+ Exec(ctx)
+ return
+}
+
+func (m MirrorSource) BuildCloneURL(url, repoType, namespace, name string) string {
+ namespace, _ = strings.CutPrefix(namespace, m.SourceName)
+ return fmt.Sprintf("%s/%ss/%s/%s.git", url, repoType, namespace, name)
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type modelStoreImpl struct {
+ db *DB
+}
+
+type ModelStore interface {
+ ByRepoIDs(ctx context.Context, repoIDs []int64) (models []Model, err error)
+ ByRepoID(ctx context.Context, repoID int64) (*Model, error)
+ ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (models []Model, total int, err error)
+ UserLikesModels(ctx context.Context, userID int64, per, page int) (models []Model, total int, err error)
+ ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (models []Model, total int, err error)
+ Count(ctx context.Context) (count int, err error)
+ PublicCount(ctx context.Context) (count int, err error)
+ Create(ctx context.Context, input Model) (*Model, error)
+ Update(ctx context.Context, input Model) (*Model, error)
+ FindByPath(ctx context.Context, namespace string, name string) (*Model, error)
+ Delete(ctx context.Context, input Model) error
+ ListByPath(ctx context.Context, paths []string) ([]Model, error)
+ ByID(ctx context.Context, id int64) (*Model, error)
+ CreateIfNotExist(ctx context.Context, input Model) (*Model, error)
+}
+
+func NewModelStore() ModelStore {
+ return &modelStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewModelStoreWithDB(db *DB) ModelStore {
+ return &modelStoreImpl{
+ db: db,
+ }
+}
+
+type Model struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ LastUpdatedAt time.Time `bun:",notnull" json:"last_updated_at"`
+ BaseModel string `bun:"," json:"base_model"`
+ times
+}
+
+func (s *modelStoreImpl) ByRepoIDs(ctx context.Context, repoIDs []int64) (models []Model, err error) {
+ err = s.db.Operator.Core.NewSelect().
+ Model(&models).
+ Relation("Repository").
+ Where("repository_id in (?)", bun.In(repoIDs)).
+ Scan(ctx)
+
+ return
+}
+
+func (s *modelStoreImpl) ByRepoID(ctx context.Context, repoID int64) (*Model, error) {
+ var m Model
+ err := s.db.Core.NewSelect().
+ Model(&m).
+ Where("repository_id = ?", repoID).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model by id, repository id: %d,error: %w", repoID, err)
+ }
+
+ return &m, nil
+}
+
+func (s *modelStoreImpl) ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (models []Model, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&models).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", username))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("model.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *modelStoreImpl) UserLikesModels(ctx context.Context, userID int64, per, page int) (models []Model, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&models).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.id in (select repo_id from user_likes where user_id=?)", userID)
+
+ query = query.Order("model.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *modelStoreImpl) ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (models []Model, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&models).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", namespace))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("model.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx, &models)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *modelStoreImpl) Count(ctx context.Context) (count int, err error) {
+ count, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&Repository{}).
+ Where("repository_type = ?", types.ModelRepo).
+ Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *modelStoreImpl) PublicCount(ctx context.Context) (count int, err error) {
+ count, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&Repository{}).
+ Where("repository_type = ?", types.DatasetRepo).
+ Where("private = ?", false).
+ Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *modelStoreImpl) Create(ctx context.Context, input Model) (*Model, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create model in db failed", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("create model in db failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *modelStoreImpl) Update(ctx context.Context, input Model) (*Model, error) {
+ _, err := s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+
+ return &input, err
+}
+
+func (s *modelStoreImpl) FindByPath(ctx context.Context, namespace string, name string) (*Model, error) {
+ resModel := new(Model)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(resModel).
+ Relation("Repository.User").
+ Relation("Repository.Mirror").
+ Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, name)).
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model,error: %w", err)
+ }
+ err = s.db.Operator.Core.NewSelect().
+ Model(resModel.Repository).
+ WherePK().
+ Relation("Tags", func(sq *bun.SelectQuery) *bun.SelectQuery {
+ return sq.Where("repository_tag.count > 0")
+ }).
+ Scan(ctx)
+ return resModel, err
+}
+
+func (s *modelStoreImpl) Delete(ctx context.Context, input Model) error {
+ res, err := s.db.Operator.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("delete model in tx failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *modelStoreImpl) ListByPath(ctx context.Context, paths []string) ([]Model, error) {
+ var models []Model
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&Model{}).
+ Relation("Repository").
+ Where("repository.path IN (?)", bun.In(paths)).
+ Scan(ctx, &models)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find models by path,error: %w", err)
+ }
+
+ var sortedModels []Model
+ for _, path := range paths {
+ for _, m := range models {
+ if m.Repository.Path == path {
+ sortedModels = append(sortedModels, m)
+ }
+ }
+ }
+
+ return sortedModels, nil
+}
+
+func (s *modelStoreImpl) ByID(ctx context.Context, id int64) (*Model, error) {
+ var model Model
+ err := s.db.Core.NewSelect().Model(&model).Relation("Repository").Where("model.id = ?", id).Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &model, err
+}
+
+func (s *modelStoreImpl) CreateIfNotExist(ctx context.Context, input Model) (*Model, error) {
+ err := s.db.Core.NewSelect().
+ Model(&input).
+ Where("repository_id = ?", input.RepositoryID).
+ Relation("Repository").
+ Scan(ctx)
+ if err == nil {
+ return &input, nil
+ }
+
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create model in db failed", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("create model in db failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type multiSyncStoreImpl struct {
+ db *DB
+}
+
+type MultiSyncStore interface {
+ Create(ctx context.Context, v SyncVersion) (*SyncVersion, error)
+ // GetAfter get N records after version in ASC order
+ GetAfter(ctx context.Context, version, limit int64) ([]SyncVersion, error)
+ // GetLatest get max sync version
+ GetLatest(ctx context.Context) (SyncVersion, error)
+ GetAfterDistinct(ctx context.Context, version int64) ([]SyncVersion, error)
+}
+
+func NewMultiSyncStore() MultiSyncStore {
+ return &multiSyncStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewMultiSyncStoreWithDB(db *DB) MultiSyncStore {
+ return &multiSyncStoreImpl{
+ db: db,
+ }
+}
+
+func (s *multiSyncStoreImpl) Create(ctx context.Context, v SyncVersion) (*SyncVersion, error) {
+ res, err := s.db.Core.NewInsert().Model(&v).Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create sync version in db failed,error:%w", err)
+ }
+
+ return &v, nil
+}
+
+// GetAfter get N records after version in ASC order
+func (s *multiSyncStoreImpl) GetAfter(ctx context.Context, version, limit int64) ([]SyncVersion, error) {
+ var vs []SyncVersion
+ err := s.db.Core.NewSelect().Model(&vs).Where("version > ?", version).
+ Order("version asc").
+ Limit(int(limit)).
+ Scan(ctx, &vs)
+ return vs, err
+}
+
+// GetLatest get max sync version
+func (s *multiSyncStoreImpl) GetLatest(ctx context.Context) (SyncVersion, error) {
+ var v SyncVersion
+ err := s.db.Core.NewSelect().Model(&v).
+ Order("version desc").
+ Limit(1).
+ Scan(ctx, &v)
+
+ return v, err
+}
+
+func (s *multiSyncStoreImpl) GetAfterDistinct(ctx context.Context, version int64) ([]SyncVersion, error) {
+ var vs []SyncVersion
+ err := s.db.Core.NewSelect().
+ ColumnExpr("DISTINCT ON (source_id, repo_path, repo_type) version, source_id, repo_path, repo_type, last_modified_at, change_log").
+ Model(&vs).
+ Where("version > ?", version).
+ Scan(ctx, &vs)
+ return vs, err
+}
+
+
+
package database
+
+import (
+ "context"
+)
+
+// Define the NamespaceStore interface
+type NamespaceStore interface {
+ FindByPath(ctx context.Context, path string) (Namespace, error)
+ Exists(ctx context.Context, path string) (bool, error)
+}
+
+type NamespaceStoreImpl struct {
+ db *DB
+}
+
+func NewNamespaceStore() NamespaceStore {
+ return &NamespaceStoreImpl{db: defaultDB}
+}
+
+func NewNamespaceStoreWithDB(db *DB) NamespaceStore {
+ return &NamespaceStoreImpl{db: db}
+}
+
+type NamespaceType string
+
+const (
+ UserNamespace NamespaceType = "user"
+ OrgNamespace NamespaceType = "organization"
+)
+
+type Namespace struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Path string `bun:",notnull" json:"path"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ NamespaceType NamespaceType `bun:",notnull" json:"namespace_type"`
+ Mirrored bool `bun:",notnull" json:"mirrored"`
+ times
+}
+
+func (s *NamespaceStoreImpl) FindByPath(ctx context.Context, path string) (namespace Namespace, err error) {
+ namespace.Path = path
+ err = s.db.Operator.Core.NewSelect().Model(&namespace).Relation("User").Where("path = ?", path).Scan(ctx)
+ return
+}
+
+func (s *NamespaceStoreImpl) Exists(ctx context.Context, path string) (exists bool, err error) {
+ var namespace Namespace
+ return s.db.Operator.Core.
+ NewSelect().
+ Model(&namespace).
+ Where("path =?", path).
+ Exists(ctx)
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/uptrace/bun"
+)
+
+type orgStoreImpl struct {
+ db *DB
+}
+
+type OrgStore interface {
+ Create(ctx context.Context, org *Organization, namepace *Namespace) (err error)
+ GetUserOwnOrgs(ctx context.Context, username string) (orgs []Organization, total int, err error)
+ Update(ctx context.Context, org *Organization) (err error)
+ Delete(ctx context.Context, path string) (err error)
+ FindByPath(ctx context.Context, path string) (org Organization, err error)
+ Exists(ctx context.Context, path string) (exists bool, err error)
+ GetUserBelongOrgs(ctx context.Context, userID int64) (orgs []Organization, err error)
+ Search(ctx context.Context, search string, per, page int) (orgs []Organization, total int, err error)
+}
+
+func NewOrgStore() OrgStore {
+ return &orgStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewOrgStoreWithDB(db *DB) OrgStore {
+ return &orgStoreImpl{
+ db: db,
+ }
+}
+
+type Organization struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Nickname string `bun:"name,notnull" json:"name"`
+ // unique name of the organization
+ Name string `bun:"path,notnull" json:"path"`
+ GitPath string `bun:",notnull" json:"git_path"`
+ Description string `json:"description"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ Homepage string `bun:"" json:"homepage,omitempty"`
+ Logo string `bun:"" json:"logo,omitempty"`
+ Verified bool `bun:"" json:"verified"`
+ OrgType string `bun:"" json:"org_type"`
+ User *User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ NamespaceID int64 `bun:",notnull" json:"namespace_id"`
+ Namespace *Namespace `bun:"rel:has-one,join:namespace_id=id" json:"namespace"`
+ times
+}
+
+func (s *orgStoreImpl) Create(ctx context.Context, org *Organization, namepace *Namespace) (err error) {
+ err = s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err = assertAffectedOneRow(tx.NewInsert().Model(org).Exec(ctx)); err != nil {
+ return err
+ }
+ namepace.NamespaceType = OrgNamespace
+ if err = assertAffectedOneRow(tx.NewInsert().Model(namepace).Exec(ctx)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return
+}
+
+func (s *orgStoreImpl) GetUserOwnOrgs(ctx context.Context, username string) (orgs []Organization, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&orgs).
+ Relation("User")
+ if username != "" {
+ query = query.
+ Join("JOIN users AS u ON u.id = organization.user_id").
+ Where("u.username =?", username)
+ }
+
+ err = query.Scan(ctx, &orgs)
+ total = len(orgs)
+ return
+}
+
+func (s *orgStoreImpl) Update(ctx context.Context, org *Organization) (err error) {
+ err = assertAffectedOneRow(s.db.Operator.Core.
+ NewUpdate().
+ Model(org).
+ WherePK().
+ Exec(ctx))
+ return
+}
+
+func (s *orgStoreImpl) Delete(ctx context.Context, path string) (err error) {
+ err = s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err = assertAffectedOneRow(
+ tx.NewDelete().
+ Model(&Organization{}).
+ Where("path = ?", path).
+ Exec(ctx)); err != nil {
+ return err
+ }
+ if err = assertAffectedOneRow(
+ tx.NewDelete().
+ Model(&Namespace{}).
+ Where("path = ?", path).
+ Exec(ctx)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return
+}
+
+func (s *orgStoreImpl) FindByPath(ctx context.Context, path string) (org Organization, err error) {
+ org.Nickname = path
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(&org).
+ Where("path =?", path).
+ Scan(ctx)
+ return
+}
+
+func (s *orgStoreImpl) Exists(ctx context.Context, path string) (exists bool, err error) {
+ var org Organization
+ exists, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&org).
+ Where("path =?", path).
+ Exists(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *orgStoreImpl) GetUserBelongOrgs(ctx context.Context, userID int64) (orgs []Organization, err error) {
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(&orgs).
+ Join("join members on members.organization_id = organization.id").
+ Where("members.user_id = ?", userID).
+ Scan(ctx, &orgs)
+ return
+}
+
+func (s *orgStoreImpl) Search(ctx context.Context, search string, per int, page int) (orgs []Organization, total int, err error) {
+ search = strings.ToLower(search)
+ query := s.db.Operator.Core.NewSelect().
+ Model(&orgs)
+ if search != "" {
+ query.Where("LOWER(name) like ? OR LOWER(path) like ?", fmt.Sprintf("%%%s%%", search), fmt.Sprintf("%%%s%%", search))
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ query.Order("id asc").Limit(per).Offset((page - 1) * per)
+ err = query.Scan(ctx, &orgs)
+ if err != nil {
+ return
+ }
+ return
+}
+
+
+
package database
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "time"
+)
+
+type Payment struct {
+ bun.BaseModel `bun:"table:payment_payment"`
+
+ PaymentUUID string `bun:",notnull,pk,skipupdate" json:"payment_uuid"`
+
+ // Transaction serial number returned by the payment channel.
+ TransactionNo string `json:"transaction_no"`
+
+ // Order number, tailored to the requirements of each channel, and must be unique within the business system.
+ // For example, in the case of a recharge, this field corresponds to the orderNo in the recharge table.
+ // For payment channels, this parameter typically corresponds to out_trade_no.
+ OrderNo string `bun:",notnull,skipupdate" json:"order_no"`
+
+ // Payment channel.
+ Channel consts.PaymentChannel `bun:",notnull,skipupdate" json:"channel"`
+
+ // Transformed into a QR code for frontend scanning payment scenarios.
+ CodeUrl string `bun:",skipupdate" json:"code_url"`
+
+ // Payment credentials used by the client to initiate a payment.
+ Credentials json.RawMessage `bun:",nullzero" json:"credentials"`
+
+ // Client IP address.
+ ClientIp string `bun:",skipupdate" json:"client_ip"`
+
+ // Total amount in the smallest currency unit (e.g., in CNY, this is expressed in cents).
+ Amount int64 `bun:",notnull,skipupdate" json:"amount"`
+
+ // 3-letter ISO currency code, represented in uppercase letters.
+ Currency string `bun:",notnull,skipupdate,default:'CNY'" json:"currency"`
+
+ // Product title, limited to a maximum of 32 Unicode characters.
+ Subject string `bun:",notnull,skipupdate" json:"subject"`
+
+ // Product description, limited to a maximum of 128 Unicode characters.
+ // Note: yeepay_wap restricts this parameter to a maximum of 100 Unicode characters;
+ // some channels of Alipay do not support special characters.
+ Body string `bun:",skipupdate" json:"body"`
+
+ // Custom fields for business-specific use cases.
+ Extra string `bun:",skipupdate" json:"extra"`
+
+ // Indicates whether the payment has been completed.
+ Paid bool `json:"paid"`
+
+ // Indicates whether the order has been revoked.
+ Reversed bool `json:"reversed"`
+
+ // Unix timestamp representing the time when the payment was completed.
+ TimePaid time.Time `bun:",nullzero" json:"time_paid"`
+
+ // Unix timestamp representing the expiration time of the order.
+ TimeExpire time.Time `bun:",nullzero" json:"time_expire"`
+
+ // Payment creation time.
+ CreatedAt time.Time `bun:",notnull,skipupdate,default:current_timestamp" json:"created_at"`
+
+ // Payment update time.
+ UpdatedAt time.Time `bun:",notnull,default:current_timestamp" json:"updated_at"`
+
+ // Error code returned in case of payment failure.
+ FailureCode string `json:"failure_code"`
+
+ // Error message or description for the payment failure.
+ FailureMsg string `json:"failure_msg"`
+}
+
+type PaymentStore struct {
+ db *DB
+}
+
+func NewPaymentStore() *PaymentStore {
+ return &PaymentStore{db: defaultDB}
+}
+
+func (ps *PaymentStore) CreatePayment(ctx context.Context, payment *Payment) error {
+ _, err := ps.db.Operator.Core.NewInsert().
+ Model(payment).
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("create payment record, error: %w", err)
+ }
+ return nil
+}
+
+func (ps *PaymentStore) GetPaymentByID(ctx context.Context, paymentUUID string) (*Payment, error) {
+ var payment Payment
+ err := ps.db.Operator.Core.NewSelect().
+ Model(&payment).
+ Where("uuid = ?", paymentUUID).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("get payment by uuid, error: %w", err)
+ }
+ return &payment, nil
+}
+
+func (ps *PaymentStore) GetPaymentByOrderNo(ctx context.Context, orderNo string) (*Payment, error) {
+ var payment Payment
+ err := ps.db.Operator.Core.NewSelect().
+ Model(&payment).
+ Where("order_no = ?", orderNo).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("get payment by orderNo, error: %w", err)
+ }
+ return &payment, nil
+}
+
+func (ps *PaymentStore) UpdatePayment(ctx context.Context, payment *Payment) error {
+ payment.UpdatedAt = time.Now()
+ _, err := ps.db.Operator.Core.NewUpdate().
+ Model(payment).
+ Where("payment_uuid = ?", payment.PaymentUUID). // 根据 UUID 定位记录
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("update payment record, error: %w", err)
+ }
+ return nil
+}
+
+func (ps *PaymentStore) ListPayments(ctx context.Context, filter *PaymentFilter) ([]*Payment, error) {
+ var payments []*Payment
+ query := ps.db.Operator.Core.NewSelect().
+ Model(&payments).
+ Order("created_at DESC")
+
+ if filter != nil {
+ if filter.UserUUID != "" {
+ query.Where("user_uuid = ?", filter.UserUUID)
+ }
+ if filter.Paid != nil {
+ query.Where("paid = ?", *filter.Paid)
+ }
+ if filter.Channel != nil {
+ query.Where("channel = ?", *filter.Channel)
+ }
+ if filter.Limit > 0 {
+ query.Limit(filter.Limit)
+ }
+ if filter.Offset > 0 {
+ query.Offset(filter.Offset)
+ }
+ }
+
+ err := query.Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list payments, error: %w", err)
+ }
+ return payments, nil
+}
+
+type PaymentFilter struct {
+ UserUUID string
+ Paid *bool
+ Channel *consts.PaymentChannel
+ Limit int
+ Offset int
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/uptrace/bun"
+)
+
+type Prompt struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id" json:"repository"`
+ times
+}
+
+type promptStoreImpl struct {
+ db *DB
+}
+
+type PromptStore interface {
+ Create(ctx context.Context, input Prompt) (*Prompt, error)
+ ByRepoIDs(ctx context.Context, repoIDs []int64) (prompts []Prompt, err error)
+ ByRepoID(ctx context.Context, repoID int64) (*Prompt, error)
+ Update(ctx context.Context, input Prompt) (err error)
+ FindByPath(ctx context.Context, namespace string, repoPath string) (*Prompt, error)
+ Delete(ctx context.Context, input Prompt) error
+ ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (prompts []Prompt, total int, err error)
+ ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (prompts []Prompt, total int, err error)
+}
+
+func NewPromptStoreWithDB(db *DB) PromptStore {
+ return &promptStoreImpl{db: db}
+}
+
+func NewPromptStore() PromptStore {
+ return &promptStoreImpl{db: defaultDB}
+}
+
+func (s *promptStoreImpl) Create(ctx context.Context, input Prompt) (*Prompt, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create prompt in db failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *promptStoreImpl) ByRepoIDs(ctx context.Context, repoIDs []int64) (prompts []Prompt, err error) {
+ q := s.db.Operator.Core.NewSelect().
+ Model(&prompts).
+ Relation("Repository").
+ Relation("Repository.User").
+ Where("repository_id in (?)", bun.In(repoIDs))
+ err = q.Scan(ctx)
+ return
+}
+
+func (s *promptStoreImpl) ByRepoID(ctx context.Context, repoID int64) (*Prompt, error) {
+ var prompt Prompt
+ err := s.db.Operator.Core.NewSelect().
+ Model(&prompt).
+ Where("repository_id = ?", repoID).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to select prompt by repository id: %d, error: %w", repoID, err)
+ }
+
+ return &prompt, nil
+}
+
+func (s *promptStoreImpl) Update(ctx context.Context, input Prompt) (err error) {
+ _, err = s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+ return
+}
+
+func (s *promptStoreImpl) FindByPath(ctx context.Context, namespace string, repoPath string) (*Prompt, error) {
+ resPrompt := new(Prompt)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(resPrompt).
+ Relation("Repository.User").
+ Where("repository.path =?", fmt.Sprintf("%s/%s", namespace, repoPath)).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find prompt: %w", err)
+ }
+ err = s.db.Operator.Core.NewSelect().
+ Model(resPrompt.Repository).
+ WherePK().
+ Relation("Tags", func(sq *bun.SelectQuery) *bun.SelectQuery {
+ return sq.Where("repository_tag.count > 0")
+ }).
+ Scan(ctx)
+ return resPrompt, err
+}
+
+func (s *promptStoreImpl) Delete(ctx context.Context, input Prompt) error {
+ res, err := s.db.Operator.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("delete prompt failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *promptStoreImpl) ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (prompts []Prompt, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&prompts).
+ Relation("Repository.User").
+ Where("username = ?", username)
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("prompt.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *promptStoreImpl) ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (prompts []Prompt, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&prompts).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", namespace))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("prompt.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx, &prompts)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/uptrace/bun"
+)
+
+type promptConversationStoreImpl struct {
+ db *DB
+}
+
+type PromptConversation struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ ConversationID string `bun:",notnull" json:"conversation_id"`
+ Title string `bun:",notnull" json:"title"`
+ times
+ Messages []PromptConversationMessage `bun:"rel:has-many,join:conversation_id=conversation_id" json:"messages"`
+}
+
+type PromptConversationMessage struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ConversationID string `bun:",notnull" json:"conversation_id"`
+ Role string `bun:",notnull" json:"role"`
+ Content string `bun:",notnull" json:"content"`
+ UserLike bool `bun:",notnull" json:"user_like"`
+ UserHate bool `bun:",notnull" json:"user_hate"`
+ times
+}
+
+type PromptConversationStore interface {
+ CreateConversation(ctx context.Context, conversation PromptConversation) error
+ SaveConversationMessage(ctx context.Context, message PromptConversationMessage) (*PromptConversationMessage, error)
+ UpdateConversation(ctx context.Context, conversation PromptConversation) error
+ FindConversationsByUserID(ctx context.Context, userID int64) ([]PromptConversation, error)
+ GetConversationByID(ctx context.Context, userID int64, uuid string, hasDetail bool) (*PromptConversation, error)
+ DeleteConversationsByID(ctx context.Context, userID int64, uuid string) error
+ LikeMessageByID(ctx context.Context, id int64) error
+ HateMessageByID(ctx context.Context, id int64) error
+}
+
+func NewPromptConversationStore() PromptConversationStore {
+ return &promptConversationStoreImpl{db: defaultDB}
+}
+
+func NewPromptConversationStoreWithDB(db *DB) PromptConversationStore {
+ return &promptConversationStoreImpl{db: db}
+}
+
+func (p *promptConversationStoreImpl) CreateConversation(ctx context.Context, conversation PromptConversation) error {
+ err := p.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err := assertAffectedOneRow(tx.NewInsert().Model(&conversation).Exec(ctx)); err != nil {
+ return fmt.Errorf("save conversation, %v, error:%w", conversation, err)
+ }
+ return nil
+ })
+ return err
+}
+
+func (p *promptConversationStoreImpl) SaveConversationMessage(ctx context.Context, message PromptConversationMessage) (*PromptConversationMessage, error) {
+ res, err := p.db.Core.NewInsert().Model(&message).Exec(ctx, &message)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("insert message, %v, error:%w", message, err)
+ }
+ return &message, nil
+}
+
+func (p *promptConversationStoreImpl) UpdateConversation(ctx context.Context, conversation PromptConversation) error {
+ res, err := p.db.Core.NewUpdate().Model(&conversation).
+ Where("user_id = ?", conversation.UserID).
+ Where("conversation_id = ?", conversation.ConversationID).
+ Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("update conversation, %v, error:%w", conversation, err)
+ }
+ return nil
+}
+
+func (p *promptConversationStoreImpl) FindConversationsByUserID(ctx context.Context, userID int64) ([]PromptConversation, error) {
+ var conversations []PromptConversation
+ err := p.db.Operator.Core.NewSelect().Model(&conversations).Where("user_id = ?", userID).Order("id desc").Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select conversation by userid %d, error: %w", userID, err)
+ }
+ return conversations, nil
+}
+
+func (p *promptConversationStoreImpl) GetConversationByID(ctx context.Context, userID int64, uuid string, hasDetail bool) (*PromptConversation, error) {
+ var conversation PromptConversation
+ q := p.db.Operator.Core.NewSelect().Model(&conversation)
+ if hasDetail {
+ q = q.Relation("Messages")
+ }
+ err := q.Where("user_id = ? and conversation_id = ?", userID, uuid).Order("id desc").Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select user conversation by userid %d, uuid %s, error: %w", userID, uuid, err)
+ }
+ return &conversation, nil
+}
+
+func (p *promptConversationStoreImpl) DeleteConversationsByID(ctx context.Context, userID int64, uuid string) error {
+ err := p.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ res, err := tx.NewDelete().Model(&PromptConversation{}).Where("user_id = ? and conversation_id = ?", userID, uuid).Exec(ctx)
+ err = assertAffectedOneRow(res, err)
+ if err != nil {
+ return fmt.Errorf("delete conversation by userid %d, %s, error: %w", userID, uuid, err)
+ }
+
+ _, err = tx.NewDelete().Model(&PromptConversationMessage{}).Where("conversation_id = ?", uuid).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("delete conversation message by uuid, %s, error:%w", uuid, err)
+ }
+ return nil
+ })
+ return err
+}
+
+func (p *promptConversationStoreImpl) LikeMessageByID(ctx context.Context, id int64) error {
+ res, err := p.db.BunDB.Exec("update prompt_conversation_messages set user_like=NOT user_like where id = ?", id)
+ if err != nil {
+ return err
+ }
+ err = assertAffectedOneRow(res, err)
+ return err
+}
+
+func (p *promptConversationStoreImpl) HateMessageByID(ctx context.Context, id int64) error {
+ res, err := p.db.BunDB.Exec("update prompt_conversation_messages set user_hate=NOT user_hate where id = ?", id)
+ if err != nil {
+ return err
+ }
+ err = assertAffectedOneRow(res, err)
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type promptPrefixStoreImpl struct {
+ db *DB
+}
+
+type PromptPrefix struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ZH string `bun:",notnull" json:"zh"`
+ EN string `bun:",notnull" json:"en"`
+}
+
+type PromptPrefixStore interface {
+ Get(ctx context.Context) (*PromptPrefix, error)
+}
+
+func NewPromptPrefixStore() PromptPrefixStore {
+ return &promptPrefixStoreImpl{db: defaultDB}
+}
+
+func NewPromptPrefixStoreWithDB(db *DB) PromptPrefixStore {
+ return &promptPrefixStoreImpl{db: db}
+}
+
+func (p *promptPrefixStoreImpl) Get(ctx context.Context) (*PromptPrefix, error) {
+ var prefix PromptPrefix
+ err := p.db.Operator.Core.NewSelect().Model(&prefix).Order("id desc").Limit(1).Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select latest prompt prefix: %w", err)
+ }
+ return &prefix, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/uptrace/bun"
+)
+
+type recomStoreImpl struct {
+ db *DB
+}
+
+type RecomStore interface {
+ // Index returns repos in descend order of score.
+ Index(ctx context.Context, page, pageSize int) ([]*RecomRepoScore, error)
+ // Upsert recom repo score
+ UpsertScore(ctx context.Context, repoID int64, score float64) error
+ LoadWeights(ctx context.Context) ([]*RecomWeight, error)
+ LoadOpWeights(ctx context.Context) ([]*RecomOpWeight, error)
+ LoadRepoOpWeights(ctx context.Context, repoIDs []int64) (map[int64]int, error)
+ UpsetOpWeights(ctx context.Context, repoID, weight int64) error
+}
+
+func NewRecomStore() RecomStore {
+ return &recomStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRecomStoreWithDB(db *DB) RecomStore {
+ return &recomStoreImpl{
+ db: db,
+ }
+}
+
+// Index returns repos in descend order of score.
+func (s *recomStoreImpl) Index(ctx context.Context, page, pageSize int) ([]*RecomRepoScore, error) {
+ items := make([]*RecomRepoScore, 0)
+ err := s.db.Operator.Core.NewSelect().Model(&RecomRepoScore{}).
+ Order("score desc").
+ Offset(page*pageSize).Limit(pageSize).
+ Scan(ctx, &items)
+ return items, err
+}
+
+// Upsert recom repo score
+func (s *recomStoreImpl) UpsertScore(ctx context.Context, repoID int64, score float64) error {
+ _, err := s.db.Operator.Core.NewInsert().
+ Model(&RecomRepoScore{
+ RepositoryID: repoID,
+ Score: score,
+ }).
+ On("CONFLICT (repository_id) DO UPDATE").
+ Exec(ctx)
+ return err
+}
+
+func (s *recomStoreImpl) LoadWeights(ctx context.Context) ([]*RecomWeight, error) {
+ weights := make([]*RecomWeight, 0)
+ err := s.db.Operator.Core.NewSelect().Model(&RecomWeight{}).Scan(ctx, &weights)
+ return weights, err
+}
+
+func (s *recomStoreImpl) LoadOpWeights(ctx context.Context) ([]*RecomOpWeight, error) {
+ weights := make([]*RecomOpWeight, 0)
+ err := s.db.Operator.Core.NewSelect().Model(&RecomOpWeight{}).Scan(ctx, &weights)
+ return weights, err
+}
+
+func (s *recomStoreImpl) LoadRepoOpWeights(ctx context.Context, repoIDs []int64) (map[int64]int, error) {
+ weights := make([]*RecomOpWeight, 0)
+ err := s.db.Operator.Core.NewSelect().Model(&RecomOpWeight{}).Where("repository_id IN (?)", bun.In(repoIDs)).
+ Column("repository_id", "weight").
+ Scan(ctx, &weights)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load repo op weights: %w", err)
+ }
+ repoWeights := make(map[int64]int)
+ for _, weight := range weights {
+ repoWeights[weight.RepositoryID] = weight.Weight
+ }
+ return repoWeights, nil
+}
+
+func (s *recomStoreImpl) UpsetOpWeights(ctx context.Context, repoID, weight int64) error {
+ _, err := s.db.Core.NewInsert().
+ Model(&RecomOpWeight{
+ RepositoryID: repoID,
+ Weight: int(weight),
+ }).
+ On("CONFLICT (repository_id) DO UPDATE").
+ Exec(ctx)
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+)
+
+type repoRelationsStoreImpl struct {
+ db *DB
+}
+
+type RepoRelationsStore interface {
+ // From gets the relationships from a repository
+ From(ctx context.Context, repoID int64) ([]*RepoRelation, error)
+ // To gets the relationships to a repository
+ To(ctx context.Context, repoID int64) ([]*RepoRelation, error)
+ // Override replaces all existing relationships from a repository to others
+ //
+ // `to` can be empty, in which case all existing relationships will be deleted
+ Override(ctx context.Context, from int64, to ...int64) error
+ // Delete removes a relationship from a repository to another
+ Delete(ctx context.Context, from, to int64) error
+}
+
+func NewRepoRelationsStore() RepoRelationsStore {
+ return &repoRelationsStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRepoRelationsStoreWithDB(db *DB) RepoRelationsStore {
+ return &repoRelationsStoreImpl{
+ db: db,
+ }
+}
+
+type RepoRelation struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ FromRepoID int64 `bun:",notnull" json:"from_repo_id"`
+ ToRepoID int64 `bun:",notnull" json:"to_repo_id"`
+}
+
+// From gets the relationships from a repository
+func (r *repoRelationsStoreImpl) From(ctx context.Context, repoID int64) ([]*RepoRelation, error) {
+ var rrs []*RepoRelation
+ err := r.db.Core.NewSelect().Model(&rrs).Where("from_repo_id = ?", repoID).Scan(ctx)
+ return rrs, err
+}
+
+// To gets the relationships to a repository
+func (r *repoRelationsStoreImpl) To(ctx context.Context, repoID int64) ([]*RepoRelation, error) {
+ var rrs []*RepoRelation
+ err := r.db.Core.NewSelect().Model(&rrs).Where("to_repo_id = ?", repoID).Scan(ctx)
+ return rrs, err
+}
+
+// Override replaces all existing relationships from a repository to others
+//
+// `to` can be empty, in which case all existing relationships will be deleted
+func (r *repoRelationsStoreImpl) Override(ctx context.Context, from int64, to ...int64) error {
+ var relations []*RepoRelation
+ for _, toRepoID := range to {
+ relations = append(relations, &RepoRelation{
+ FromRepoID: from,
+ ToRepoID: toRepoID,
+ })
+ }
+
+ tx, err := r.db.Core.BeginTx(ctx, nil)
+ if err != nil {
+ return fmt.Errorf("failed to begin transaction: %w", err)
+ }
+ _, err = tx.NewDelete().Model((*RepoRelation)(nil)).Where("from_repo_id = ?", from).Exec(ctx)
+ if err != nil {
+ if er := tx.Rollback(); er != nil {
+ slog.Error("rollback failed", "error", er)
+ }
+ return fmt.Errorf("failed to delete existing relations: %w", err)
+ }
+
+ if len(relations) > 0 {
+ _, err = tx.NewInsert().Model(&relations).
+ Exec(ctx)
+ if err != nil {
+ if er := tx.Rollback(); er != nil {
+ slog.Error("tx rollback failed", "error", er)
+ }
+ return fmt.Errorf("failed to insert relations: %w", err)
+ }
+ }
+
+ return tx.Commit()
+}
+
+// Delete removes a relationship from a repository to another
+func (r *repoRelationsStoreImpl) Delete(ctx context.Context, from, to int64) error {
+ result, err := r.db.Core.NewDelete().
+ Model((*RepoRelation)(nil)).
+ Where("from_repo_id = ? and to_repo_id = ?", from, to).
+ Exec(ctx)
+ return assertAffectedOneRow(result, err)
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strings"
+ "time"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var RepositorySourceAndPrefixMapping = map[types.RepositorySource]string{
+ types.HuggingfaceSource: types.HuggingfacePrefix,
+ types.OpenCSGSource: types.OpenCSGPrefix,
+ types.LocalSource: "",
+}
+
+type repoStoreImpl struct {
+ db *DB
+}
+
+type RepoStore interface {
+ // CreateRepoTx(ctx context.Context, tx bun.Tx, input Repository) (*Repository, error)
+ CreateRepo(ctx context.Context, input Repository) (*Repository, error)
+ UpdateRepo(ctx context.Context, input Repository) (*Repository, error)
+ DeleteRepo(ctx context.Context, input Repository) error
+ Find(ctx context.Context, owner, repoType, repoName string) (*Repository, error)
+ FindById(ctx context.Context, id int64) (*Repository, error)
+ FindByIds(ctx context.Context, ids []int64, opts ...SelectOption) ([]*Repository, error)
+ FindByPath(ctx context.Context, repoType types.RepositoryType, namespace, name string) (*Repository, error)
+ FindByGitPath(ctx context.Context, path string) (*Repository, error)
+ FindByGitPaths(ctx context.Context, paths []string, opts ...SelectOption) ([]*Repository, error)
+ Exists(ctx context.Context, repoType types.RepositoryType, namespace string, name string) (bool, error)
+ All(ctx context.Context) ([]*Repository, error)
+ UpdateRepoFileDownloads(ctx context.Context, repo *Repository, date time.Time, clickDownloadCount int64) (err error)
+ UpdateRepoCloneDownloads(ctx context.Context, repo *Repository, date time.Time, cloneCount int64) (err error)
+ UpdateDownloads(ctx context.Context, repo *Repository) error
+ Tags(ctx context.Context, repoID int64) (tags []Tag, err error)
+ TagsWithCategory(ctx context.Context, repoID int64, category string) (tags []Tag, err error)
+ // TagIDs get tag ids by repo id, if category is not empty, return only tags of the category
+ TagIDs(ctx context.Context, repoID int64, category string) (tagIDs []int64, err error)
+ SetUpdateTimeByPath(ctx context.Context, repoType types.RepositoryType, namespace, name string, update time.Time) error
+ PublicToUser(ctx context.Context, repoType types.RepositoryType, userIDs []int64, filter *types.RepoFilter, per, page int, isAdmin bool) (repos []*Repository, count int, err error)
+ IsMirrorRepo(ctx context.Context, repoType types.RepositoryType, namespace, name string) (bool, error)
+ ListRepoPublicToUserByRepoIDs(ctx context.Context, repoType types.RepositoryType, userID int64, search, sort string, per, page int, repoIDs []int64) (repos []*Repository, count int, err error)
+ WithMirror(ctx context.Context, per, page int) (repos []Repository, count int, err error)
+ CleanRelationsByRepoID(ctx context.Context, repoId int64) error
+ BatchCreateRepoTags(ctx context.Context, repoTags []RepositoryTag) error
+ DeleteAllFiles(ctx context.Context, repoID int64) error
+ DeleteAllTags(ctx context.Context, repoID int64) error
+ UpdateOrCreateRepo(ctx context.Context, input Repository) (*Repository, error)
+ UpdateLicenseByTag(ctx context.Context, repoID int64) error
+ CountByRepoType(ctx context.Context, repoType types.RepositoryType) (int, error)
+ GetRepoWithoutRuntimeByID(ctx context.Context, rfID int64, paths []string) ([]Repository, error)
+ GetRepoWithRuntimeByID(ctx context.Context, rfID int64, paths []string) ([]Repository, error)
+ BatchGet(ctx context.Context, repoType types.RepositoryType, lastRepoID int64, batch int) ([]Repository, error)
+ FindWithBatch(ctx context.Context, batchSize, batch int) ([]Repository, error)
+ ByUser(ctx context.Context, userID int64) ([]Repository, error)
+ FindByRepoSourceWithBatch(ctx context.Context, repoSource types.RepositorySource, batchSize, batch int) ([]Repository, error)
+}
+
+func NewRepoStore() RepoStore {
+ return &repoStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRepoStoreWithDB(db *DB) RepoStore {
+ return &repoStoreImpl{
+ db: db,
+ }
+}
+
+type Repository struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ Path string `bun:",notnull" json:"path"`
+ GitPath string `bun:",notnull" json:"git_path"`
+ Name string `bun:",notnull" json:"name"`
+ Nickname string `bun:",notnull" json:"nickname"`
+ Description string `bun:",nullzero" json:"description"`
+ Private bool `bun:",notnull" json:"private"`
+ // Depreated
+ Labels string `bun:",nullzero" json:"labels"`
+ License string `bun:",nullzero" json:"license"`
+ // Depreated
+ Readme string `bun:",nullzero" json:"readme"`
+ DefaultBranch string `bun:",notnull" json:"default_branch"`
+ LfsFiles []LfsFile `bun:"rel:has-many,join:id=repository_id" json:"-"`
+ Likes int64 `bun:",nullzero" json:"likes"`
+ DownloadCount int64 `bun:",nullzero" json:"download_count"`
+ Downloads []RepositoryDownload `bun:"rel:has-many,join:id=repository_id" json:"downloads"`
+ Tags []Tag `bun:"m2m:repository_tags,join:Repository=Tag" json:"tags"`
+ Mirror Mirror `bun:"rel:has-one,join:id=repository_id" json:"mirror"`
+ RepositoryType types.RepositoryType `bun:",notnull" json:"repository_type"`
+ HTTPCloneURL string `bun:",nullzero" json:"http_clone_url"`
+ SSHCloneURL string `bun:",nullzero" json:"ssh_clone_url"`
+ Source types.RepositorySource `bun:",nullzero,default:'local'" json:"source"`
+ SyncStatus types.RepositorySyncStatus `bun:",nullzero" json:"sync_status"`
+ SensitiveCheckStatus types.SensitiveCheckStatus `bun:",default:0" json:"sensitive_check_status"`
+ // updated_at timestamp will be updated only if files changed
+ times
+}
+
+// NamespaceAndName returns namespace and name by parsing repository path
+func (r Repository) NamespaceAndName() (namespace string, name string) {
+ fields := strings.Split(r.Path, "/")
+ return fields[0], fields[1]
+}
+
+type RepositoryTag struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepositoryID int64 `bun:",notnull" json:"repository_id"`
+ TagID int64 `bun:",notnull" json:"tag_id"`
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id"`
+ Tag *Tag `bun:"rel:belongs-to,join:tag_id=id"`
+ /*
+ for meta tags parsed from README.md file, count is alway 1
+
+ for Library tags, count means how many a kind of library file (e.g. *.ONNX file) exists in the repository
+ */
+ Count int32 `bun:",default:1" json:"count"`
+}
+
+func (r Repository) PathWithOutPrefix() string {
+ return strings.TrimPrefix(r.Path, RepositorySourceAndPrefixMapping[r.Source])
+
+}
+
+// func (s *repoStoreImpl) CreateRepoTx(ctx context.Context, tx bun.Tx, input Repository) (*Repository, error) {
+// res, err := tx.NewInsert().Model(&input).Exec(ctx)
+// if err := assertAffectedOneRow(res, err); err != nil {
+// return nil, fmt.Errorf("create repository in tx failed,error:%w", err)
+// }
+
+// return &input, nil
+// }
+
+func (s *repoStoreImpl) CreateRepo(ctx context.Context, input Repository) (*Repository, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create repository in tx failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *repoStoreImpl) UpdateRepo(ctx context.Context, input Repository) (*Repository, error) {
+ _, err := s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+
+ return &input, err
+}
+
+func (s *repoStoreImpl) DeleteRepo(ctx context.Context, input Repository) error {
+ _, err := s.db.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+
+ return err
+}
+
+func (s *repoStoreImpl) Find(ctx context.Context, owner, repoType, repoName string) (*Repository, error) {
+ var err error
+ repo := &Repository{}
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(repo).
+ Where("LOWER(git_path) = LOWER(?)", fmt.Sprintf("%ss_%s/%s", repoType, owner, repoName)).
+ Limit(1).
+ Scan(ctx)
+ return repo, err
+}
+
+func (s *repoStoreImpl) FindById(ctx context.Context, id int64) (*Repository, error) {
+ resRepo := new(Repository)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(resRepo).
+ Where("id =?", id).
+ Scan(ctx)
+ return resRepo, err
+}
+
+func (s *repoStoreImpl) FindByIds(ctx context.Context, ids []int64, opts ...SelectOption) ([]*Repository, error) {
+ repos := make([]*Repository, 0)
+ q := s.db.Operator.Core.
+ NewSelect()
+ for _, opt := range opts {
+ opt.Appply(q)
+ }
+ err := q.
+ Model(&repos).
+ Where("id in (?)", bun.In(ids)).
+ Scan(ctx)
+ return repos, err
+}
+
+func (s *repoStoreImpl) FindByPath(ctx context.Context, repoType types.RepositoryType, namespace, name string) (*Repository, error) {
+ resRepo := new(Repository)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(resRepo).
+ Where("LOWER(git_path) = LOWER(?)", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)).
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return resRepo, err
+}
+
+func (s *repoStoreImpl) FindByGitPath(ctx context.Context, path string) (*Repository, error) {
+ resRepo := new(Repository)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(resRepo).
+ Where("LOWER(git_path) = LOWER(?)", path).
+ Scan(ctx)
+ return resRepo, err
+}
+
+func (s *repoStoreImpl) FindByGitPaths(ctx context.Context, paths []string, opts ...SelectOption) ([]*Repository, error) {
+ for i := range paths {
+ paths[i] = strings.ToLower(paths[i])
+ }
+ repos := make([]*Repository, 0)
+ q := s.db.Operator.Core.
+ NewSelect()
+ for _, opt := range opts {
+ opt.Appply(q)
+ }
+ err := q.Model(&repos).
+ Where("LOWER(git_path) in (?)", bun.In(paths)).
+ Scan(ctx)
+ return repos, err
+}
+
+func (s *repoStoreImpl) Exists(ctx context.Context, repoType types.RepositoryType, namespace string, name string) (bool, error) {
+ return s.db.Operator.Core.NewSelect().Model((*Repository)(nil)).
+ Where("LOWER(git_path) = LOWER(?)", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)).
+ Exists(ctx)
+}
+
+func (s *repoStoreImpl) All(ctx context.Context) ([]*Repository, error) {
+ repos := make([]*Repository, 0)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&repos).
+ Scan(ctx)
+ return repos, err
+}
+
+func (s *repoStoreImpl) UpdateRepoFileDownloads(ctx context.Context, repo *Repository, date time.Time, clickDownloadCount int64) (err error) {
+ rd := new(RepositoryDownload)
+ err = s.db.Operator.Core.NewSelect().
+ Model(rd).
+ Where("date = ? AND repository_id = ?", date.Format("2006-01-02"), repo.ID).
+ Scan(ctx)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return
+ }
+
+ if errors.Is(err, sql.ErrNoRows) {
+ rd.ClickDownloadCount = clickDownloadCount
+ rd.Date = date
+ rd.RepositoryID = repo.ID
+ err = s.db.Operator.Core.NewInsert().
+ Model(rd).
+ Scan(ctx)
+ if err != nil {
+ return
+ }
+ } else {
+ rd.ClickDownloadCount = rd.ClickDownloadCount + clickDownloadCount
+ query := s.db.Operator.Core.NewUpdate().
+ Model(rd).
+ WherePK()
+ slog.Debug(query.String())
+
+ _, err = query.Exec(ctx)
+ if err != nil {
+ return
+ }
+ }
+ err = s.UpdateDownloads(ctx, repo)
+ if err != nil {
+ return
+ }
+
+ return
+}
+
+func (s *repoStoreImpl) UpdateRepoCloneDownloads(ctx context.Context, repo *Repository, date time.Time, cloneCount int64) (err error) {
+ rd := new(RepositoryDownload)
+ err = s.db.Operator.Core.NewSelect().
+ Model(rd).
+ Where("date = ? AND repository_id = ?", date.Format("2006-01-02"), repo.ID).
+ Scan(ctx)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return
+ }
+
+ if errors.Is(err, sql.ErrNoRows) {
+ rd.CloneCount = cloneCount
+ rd.Date = date
+ rd.RepositoryID = repo.ID
+ err = s.db.Operator.Core.NewInsert().
+ Model(rd).
+ Scan(ctx)
+ if err != nil {
+ return
+ }
+ } else {
+ rd.CloneCount = cloneCount
+ query := s.db.Operator.Core.NewUpdate().
+ Model(rd).
+ WherePK()
+ slog.Debug(query.String())
+
+ _, err = query.Exec(ctx)
+ if err != nil {
+ return
+ }
+ }
+ err = s.UpdateDownloads(ctx, repo)
+ if err != nil {
+ return
+ }
+
+ return
+}
+
+func (s *repoStoreImpl) UpdateDownloads(ctx context.Context, repo *Repository) error {
+ var downloadCount int64
+ err := s.db.Operator.Core.NewSelect().
+ ColumnExpr("(SUM(clone_count)+SUM(click_download_count)) AS total_count").
+ Model(&RepositoryDownload{}).
+ Where("repository_id=?", repo.ID).
+ Scan(ctx, &downloadCount)
+ if err != nil {
+ return err
+ }
+ repo.DownloadCount = downloadCount
+ _, err = s.db.Operator.Core.NewUpdate().
+ Model(repo).
+ WherePK().
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *repoStoreImpl) Tags(ctx context.Context, repoID int64) (tags []Tag, err error) {
+ query := s.db.Operator.Core.NewSelect().
+ ColumnExpr("tags.*").
+ Model(&RepositoryTag{}).
+ Join("JOIN tags ON repository_tag.tag_id = tags.id").
+ Where("repository_tag.repository_id = ?", repoID).
+ Where("repository_tag.count > 0")
+ err = query.Scan(ctx, &tags)
+ return
+}
+func (s *repoStoreImpl) TagsWithCategory(ctx context.Context, repoID int64, category string) (tags []Tag, err error) {
+ query := s.db.Operator.Core.NewSelect().
+ ColumnExpr("tags.*").
+ Model(&RepositoryTag{}).
+ Join("JOIN tags ON repository_tag.tag_id = tags.id").
+ Where("repository_tag.repository_id = ?", repoID).
+ Where("repository_tag.count > 0").
+ Where("tags.category = ?", category)
+ err = query.Scan(ctx, &tags)
+ return
+}
+
+// TagIDs get tag ids by repo id, if category is not empty, return only tags of the category
+func (s *repoStoreImpl) TagIDs(ctx context.Context, repoID int64, category string) (tagIDs []int64, err error) {
+ query := s.db.Operator.Core.NewSelect().
+ Model(&RepositoryTag{}).
+ Join("JOIN tags ON repository_tag.tag_id = tags.id").
+ Where("repository_id = ?", repoID)
+ if len(category) > 0 {
+ query.Where("tags.category = ?", category)
+ }
+ query.Column("repository_tag.tag_id")
+ err = query.Scan(ctx, &tagIDs)
+ return tagIDs, err
+}
+
+func (s *repoStoreImpl) SetUpdateTimeByPath(ctx context.Context, repoType types.RepositoryType, namespace, name string, update time.Time) error {
+ repo := new(Repository)
+ repo.UpdatedAt = update
+ _, err := s.db.Operator.Core.NewUpdate().Model(repo).
+ Column("updated_at").
+ Where("LOWER(git_path) = LOWER(?)", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)).
+ Exec(ctx)
+ return err
+}
+
+func (s *repoStoreImpl) PublicToUser(ctx context.Context, repoType types.RepositoryType, userIDs []int64, filter *types.RepoFilter, per, page int, isAdmin bool) (repos []*Repository, count int, err error) {
+ q := s.db.Operator.Core.
+ NewSelect().
+ Column("repository.*").
+ Model(&repos).
+ Relation("Tags")
+
+ q.Where("repository.repository_type = ?", repoType)
+
+ if !isAdmin {
+ if len(userIDs) > 0 {
+ q.Where("repository.private = ? or repository.user_id in (?)", false, bun.In(userIDs))
+ } else {
+ q.Where("repository.private = ?", false)
+ }
+ }
+
+ if filter.Source != "" {
+ q.Where("repository.source = ?", filter.Source)
+ }
+
+ if filter.Search != "" {
+ filter.Search = strings.ToLower(filter.Search)
+ q.Where(
+ "LOWER(repository.path) like ? or LOWER(repository.description) like ? or LOWER(repository.nickname) like ?",
+ fmt.Sprintf("%%%s%%", filter.Search),
+ fmt.Sprintf("%%%s%%", filter.Search),
+ fmt.Sprintf("%%%s%%", filter.Search),
+ )
+ }
+ if len(filter.Tags) > 0 {
+ for i, tag := range filter.Tags {
+ var asRepoTag = fmt.Sprintf("%s%d", "rt", i)
+ var asTag = fmt.Sprintf("%s%d", "ts", i)
+ q.Join(fmt.Sprintf("JOIN repository_tags AS %s ON repository.id = %s.repository_id", asRepoTag, asRepoTag)).
+ Join(fmt.Sprintf("JOIN tags AS %s ON %s.tag_id = %s.id", asTag, asRepoTag, asTag))
+ q.Where(fmt.Sprintf("%s.category = ? AND %s.name = ?", asTag, asTag), tag.Category, tag.Name)
+ }
+
+ }
+
+ count, err = q.Count(ctx)
+ if err != nil {
+ return
+ }
+
+ if filter.Sort == "trending" {
+ q.Join("Left Join recom_repo_scores on repository.id = recom_repo_scores.repository_id")
+ q.Join("Left Join recom_op_weights on repository.id = recom_op_weights.repository_id")
+ q.ColumnExpr(`COALESCE(recom_repo_scores.score, 0)+COALESCE(recom_op_weights.weight, 0) AS popularity`)
+ }
+
+ err = q.Order(sortBy[filter.Sort]).
+ Limit(per).Offset((page - 1) * per).
+ Scan(ctx)
+
+ return
+}
+
+func (s *repoStoreImpl) IsMirrorRepo(ctx context.Context, repoType types.RepositoryType, namespace, name string) (bool, error) {
+ var result struct {
+ Exists bool `bun:"exists"`
+ }
+
+ err := s.db.Operator.Core.NewSelect().
+ ColumnExpr("EXISTS(SELECT 1 FROM mirrors WHERE mirrors.repository_id = repositories.id) AS exists").
+ Table("repositories").
+ Where("LOWER(repositories.git_path) = LOWER(?)", fmt.Sprintf("%ss_%s/%s", repoType, namespace, name)).
+ Limit(1).
+ Scan(ctx, &result)
+ if err != nil {
+ return false, err
+ }
+
+ return result.Exists, nil
+}
+
+func (s *repoStoreImpl) ListRepoPublicToUserByRepoIDs(ctx context.Context, repoType types.RepositoryType, userID int64, search, sort string, per, page int, repoIDs []int64) (repos []*Repository, count int, err error) {
+ q := s.db.Operator.Core.
+ NewSelect().
+ Column("repository.*").
+ Model(&repos).
+ Relation("Tags")
+
+ q.Where("repository.repository_type = ?", repoType)
+ q.Where("repository.private = ? or repository.user_id = ?", false, userID)
+ q.Where("id in (?)", bun.In(repoIDs))
+
+ if search != "" {
+ search = strings.ToLower(search)
+ q.Where(
+ "LOWER(repository.path) like ? or LOWER(repository.description) like ? or LOWER(repository.nickname) like ?",
+ fmt.Sprintf("%%%s%%", search),
+ fmt.Sprintf("%%%s%%", search),
+ fmt.Sprintf("%%%s%%", search),
+ )
+ }
+
+ count, err = q.Count(ctx)
+ if err != nil {
+ return
+ }
+
+ orderBy := "path"
+
+ if sort != "" {
+ if sort == "trending" {
+ q.Join("Left Join recom_repo_scores on repository.id = recom_repo_scores.repository_id")
+ q.Join("Left Join recom_op_weights on repository.id = recom_op_weights.repository_id")
+ q.ColumnExpr(`COALESCE(recom_repo_scores.score, 0)+COALESCE(recom_op_weights.weight, 0) AS popularity`)
+ }
+ sortByStr, exits := sortBy[sort]
+ if exits {
+ orderBy = sortByStr
+ }
+ }
+
+ err = q.Order(orderBy).
+ Limit(per).Offset((page - 1) * per).
+ Scan(ctx)
+
+ return
+}
+
+func (s *repoStoreImpl) WithMirror(ctx context.Context, per, page int) (repos []Repository, count int, err error) {
+ q := s.db.Operator.Core.NewSelect().
+ Model(&repos).
+ Relation("Mirror").
+ Where("mirror.id is not null")
+ count, err = q.Count(ctx)
+ if err != nil {
+ return
+ }
+ err = q.Limit(per).
+ Offset((page - 1) * per).
+ Scan(ctx)
+
+ if err != nil {
+ return
+ }
+
+ return
+}
+
+func (s *repoStoreImpl) CleanRelationsByRepoID(ctx context.Context, repoId int64) error {
+ err := s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if _, err := tx.Exec("delete from repositories_runtime_frameworks where repo_id=?", repoId); err != nil {
+ return err
+ }
+
+ if _, err := tx.Exec("delete from user_likes where repo_id=?", repoId); err != nil {
+ return err
+ }
+ return nil
+ })
+ return err
+}
+
+func (s *repoStoreImpl) BatchCreateRepoTags(ctx context.Context, repoTags []RepositoryTag) error {
+ result, err := s.db.Operator.Core.NewInsert().
+ Model(&repoTags).
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return assertAffectedXRows(int64(len(repoTags)), result, err)
+}
+
+func (s *repoStoreImpl) DeleteAllFiles(ctx context.Context, repoID int64) error {
+ _, err := s.db.Operator.Core.NewDelete().
+ Model(&File{}).
+ Where("repository_id = ?", repoID).
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *repoStoreImpl) DeleteAllTags(ctx context.Context, repoID int64) error {
+ _, err := s.db.Operator.Core.NewDelete().
+ Model(&RepositoryTag{}).
+ Where("repository_id = ?", repoID).
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (s *repoStoreImpl) UpdateOrCreateRepo(ctx context.Context, input Repository) (*Repository, error) {
+ input.UpdatedAt = time.Now()
+ _, err := s.db.Core.NewUpdate().
+ Model(&input).
+ Where("path = ? and repository_type = ?", input.Path, input.RepositoryType).
+ Returning("*").
+ Exec(ctx, &input)
+ if err == nil {
+ return &input, nil
+ }
+
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create repository in tx failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *repoStoreImpl) UpdateLicenseByTag(ctx context.Context, repoID int64) error {
+ var tag Tag
+ err := s.db.Core.NewSelect().
+ Model(&tag).
+ Join("join repository_tags on tag.id = repository_tags.tag_id").
+ Join("join repositories on repositories.id = repository_tags.repository_id").
+ Where("repository_tags.repository_id = ? and tag.category = ?", repoID, "license").
+ Scan(ctx)
+ if err != nil {
+ return err
+ }
+ if tag.Name != "" {
+ repo, err := s.FindById(ctx, repoID)
+ if err != nil {
+ return err
+ }
+ repo.License = tag.Name
+ _, err = s.UpdateRepo(ctx, *repo)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (s *repoStoreImpl) CountByRepoType(ctx context.Context, repoType types.RepositoryType) (int, error) {
+ return s.db.Core.NewSelect().Model(&Repository{}).Where("repository_type = ?", repoType).Count(ctx)
+}
+
+func (s *repoStoreImpl) GetRepoWithoutRuntimeByID(ctx context.Context, rfID int64, paths []string) ([]Repository, error) {
+ var res []Repository
+ q := s.db.Operator.Core.NewSelect().Model(&res)
+ if len(paths) > 0 {
+ q.Where("path in (?)", bun.In(paths))
+ }
+ err := q.Where("repository_type = ?", types.ModelRepo).
+ Where("id not in (select repo_id from repositories_runtime_frameworks where runtime_framework_id = ?)", rfID).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select repos without runtime failed, %w", err)
+ }
+ return res, nil
+}
+
+func (s *repoStoreImpl) GetRepoWithRuntimeByID(ctx context.Context, rfID int64, paths []string) ([]Repository, error) {
+ var res []Repository
+ q := s.db.Operator.Core.NewSelect().Model(&res)
+ if len(paths) > 0 {
+ q.Where("path in (?)", bun.In(paths))
+ }
+ err := q.Where("repository_type = ?", types.ModelRepo).
+ Where("id in (select repo_id from repositories_runtime_frameworks where runtime_framework_id = ?)", rfID).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select repos with runtime failed, %w", err)
+ }
+ return res, nil
+}
+
+func (s *repoStoreImpl) BatchGet(ctx context.Context, repoType types.RepositoryType, lastRepoID int64, batch int) ([]Repository, error) {
+ var res []Repository
+ q := s.db.Operator.Core.NewSelect().Model(&res)
+ if lastRepoID > 0 {
+ q.Where("id > ?", lastRepoID)
+ }
+ err := q.Where("repository_type = ? and sensitive_check_status = ?", repoType, types.SensitiveCheckPending).
+ Order("id ASC").
+ Limit(batch).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("select repos failed, last_repo_id: %d, batch: %d, %w", lastRepoID, batch, err)
+ }
+ return res, nil
+}
+
+func (s *repoStoreImpl) FindWithBatch(ctx context.Context, batchSize, batch int) ([]Repository, error) {
+ var res []Repository
+ err := s.db.Operator.Core.NewSelect().
+ Model(&res).
+ Order("id desc").
+ Limit(batchSize).
+ Offset(batchSize * batch).
+ Scan(ctx)
+ return res, err
+}
+
+func (s *repoStoreImpl) ByUser(ctx context.Context, userID int64) ([]Repository, error) {
+ var repos []Repository
+ err := s.db.Operator.Core.NewSelect().Model(&repos).Where("user_id = ?", userID).Scan(ctx)
+ return repos, err
+}
+
+func (s *repoStoreImpl) FindByRepoSourceWithBatch(ctx context.Context, repoSource types.RepositorySource, batchSize, batch int) ([]Repository, error) {
+ var res []Repository
+ err := s.db.Operator.Core.NewSelect().
+ Model(&res).
+ Where("source = ?", repoSource).
+ Order("id desc").
+ Limit(batchSize).
+ Offset(batchSize * batch).
+ Scan(ctx)
+ return res, err
+}
+
+
+
package database
+
+import (
+ "context"
+ "log/slog"
+ "time"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type RepositoryFile struct {
+ ID int64 `bun:",pk,autoincrement" `
+ RepositoryID int64 `bun:",notnull" `
+ Path string `bun:",notnull" `
+ FileType string `bun:",notnull" `
+ Size int64 `bun:",nullzero" `
+ LastModify time.Time `bun:",nullzero" `
+ CommitSha string `bun:",nullzero" `
+ LfsRelativePath string `bun:",nullzero" `
+ Branch string `bun:",nullzero" `
+ Repository *Repository `bun:"rel:belongs-to,join:repository_id=id"`
+}
+
+type repoFileStoreImpl struct {
+ db *DB
+}
+
+type RepoFileStore interface {
+ Create(ctx context.Context, file *RepositoryFile) error
+ BatchGet(ctx context.Context, repoID, lastRepoFileID, batch int64) ([]*RepositoryFile, error)
+ BatchGetUnchcked(ctx context.Context, repoID, lastRepoFileID, batch int64) ([]*RepositoryFile, error)
+ Exists(ctx context.Context, file RepositoryFile) (bool, error)
+ ExistsSensitiveCheckRecord(ctx context.Context, repoID int64, branch string, status types.SensitiveCheckStatus) (bool, error)
+}
+
+func NewRepoFileStore() RepoFileStore {
+ return &repoFileStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRepoFileStoreWithDB(db *DB) RepoFileStore {
+ return &repoFileStoreImpl{
+ db: db,
+ }
+}
+
+func (s *repoFileStoreImpl) Create(ctx context.Context, file *RepositoryFile) error {
+ _, err := s.db.Operator.Core.NewInsert().Model(file).Exec(ctx)
+ return err
+}
+
+func (s *repoFileStoreImpl) BatchGet(ctx context.Context, repoID, lastRepoFileID, batch int64) ([]*RepositoryFile, error) {
+ files := make([]*RepositoryFile, 0, batch)
+ err := s.db.Operator.Core.NewSelect().
+ Model(&files).
+ Relation("Repository").
+ Where("repository.id = ?", repoID).
+ Where("repository_file.id > ? ", lastRepoFileID).
+ Order("repository_file.id ASC").
+ Limit(int(batch)).
+ Scan(ctx)
+ return files, err
+}
+
+func (s *repoFileStoreImpl) BatchGetUnchcked(ctx context.Context, repoID, lastRepoFileID, batch int64) ([]*RepositoryFile, error) {
+ files := make([]*RepositoryFile, 0, batch)
+ err := s.db.Operator.Core.NewSelect().
+ Model(&files).
+ Relation("Repository").
+ Join("LEFT JOIN repository_file_checks rfc ON rfc.repo_file_id = repository_file.id").
+ Where("repository.id = ?", repoID).
+ Where("repository_file.id > ? and (rfc.status is null or rfc.status = 0)", lastRepoFileID).
+ Order("repository_file.id ASC").
+ Limit(int(batch)).
+ Scan(ctx)
+ return files, err
+}
+
+func (s *repoFileStoreImpl) Exists(ctx context.Context, file RepositoryFile) (bool, error) {
+ slog.Debug("file", slog.Any("file", file))
+ return s.db.Operator.Core.NewSelect().Model(&file).
+ Where("path = ? and repository_id = ? and branch = ? and COALESCE(commit_sha, '') = ?", file.Path, file.RepositoryID, file.Branch, file.CommitSha).
+ Exists(ctx)
+}
+
+func (s *repoFileStoreImpl) ExistsSensitiveCheckRecord(ctx context.Context, repoID int64, branch string, status types.SensitiveCheckStatus) (bool, error) {
+ return s.db.Operator.Core.NewSelect().Model(&RepositoryFileCheck{}).
+ Join("INNER JOIN repository_files rf ON rf.id = repository_file_check.repo_file_id").
+ Where("rf.repository_id = ? and rf.branch = ? and repository_file_check.status = ?", repoID, branch, status).
+ Exists(ctx)
+}
+
+
+
package database
+
+import (
+ "context"
+ "time"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+// RepositoryFileCheck is the sensitive check record for a repository file
+type RepositoryFileCheck struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RepoFileID int64 `bun:"," json:"repo_file_id"`
+ Status types.SensitiveCheckStatus `bun:",nullzero" json:"status"`
+ Message string `bun:",nullzero" json:"message"`
+ CreatedAt time.Time `bun:"created_at,notnull,default:current_timestamp" json:"created_at"`
+ //uuid for async check task
+ TaskID string `bun:",nullzero" json:"task_id"`
+}
+
+type repoFileCheckStoreImpl struct {
+ db *DB
+}
+
+type RepoFileCheckStore interface {
+ Create(ctx context.Context, history RepositoryFileCheck) error
+ Upsert(ctx context.Context, history RepositoryFileCheck) error
+}
+
+func NewRepoFileCheckStore() RepoFileCheckStore {
+ return &repoFileCheckStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRepoFileCheckStoreWithDB(db *DB) RepoFileCheckStore {
+ return &repoFileCheckStoreImpl{
+ db: db,
+ }
+}
+
+func (s *repoFileCheckStoreImpl) Create(ctx context.Context, history RepositoryFileCheck) error {
+ _, err := s.db.Operator.Core.NewInsert().Model(&history).Exec(ctx)
+ return err
+}
+
+func (s *repoFileCheckStoreImpl) Upsert(ctx context.Context, history RepositoryFileCheck) error {
+ _, err := s.db.Operator.Core.NewInsert().Model(&history).
+ On("CONFLICT (repo_file_id) DO UPDATE").
+ Exec(ctx)
+ return err
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type RepositoriesRuntimeFrameworkStore interface {
+ ListByRuntimeFrameworkID(ctx context.Context, runtimeFrameworkID int64, deployType int) ([]RepositoriesRuntimeFramework, error)
+ Add(ctx context.Context, runtimeFrameworkID, repoID int64, deployType int) error
+ Delete(ctx context.Context, runtimeFrameworkID, repoID int64, deployType int) error
+ DeleteByRepoID(ctx context.Context, repoID int64) error
+ GetByIDsAndType(ctx context.Context, runtimeFrameworkID, repoID int64, deployType int) ([]RepositoriesRuntimeFramework, error)
+ ListRepoIDsByType(ctx context.Context, deployType int) ([]RepositoriesRuntimeFramework, error)
+ GetByRepoIDsAndType(ctx context.Context, repoID int64, deployType int) ([]RepositoriesRuntimeFramework, error)
+ GetByRepoIDs(ctx context.Context, repoID int64) ([]RepositoriesRuntimeFramework, error)
+}
+
+type repositoriesRuntimeFrameworkStoreImpl struct {
+ db *DB
+}
+
+func NewRepositoriesRuntimeFramework() RepositoriesRuntimeFrameworkStore {
+ return &repositoriesRuntimeFrameworkStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRepositoriesRuntimeFrameworkWithDB(db *DB) RepositoriesRuntimeFrameworkStore {
+ return &repositoriesRuntimeFrameworkStoreImpl{
+ db: db,
+ }
+}
+
+type RepositoriesRuntimeFramework struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RuntimeFrameworkID int64 `bun:",notnull" json:"runtime_framework_id"`
+ RuntimeFramework *RuntimeFramework `bun:"rel:belongs-to,join:runtime_framework_id=id" json:"runtime_framework"`
+ RepoID int64 `bun:",notnull" json:"repo_id"`
+ Type int `bun:",notnull" json:"type"` // 0-space, 1-inference, 2-finetune
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) ListByRuntimeFrameworkID(ctx context.Context, runtimeFrameworkID int64, deployType int) ([]RepositoriesRuntimeFramework, error) {
+ var result []RepositoriesRuntimeFramework
+ _, err := m.db.Operator.Core.
+ NewSelect().
+ Model(&result).
+ Where("type = ? and runtime_framework_id = ?", deployType, runtimeFrameworkID).Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) Add(ctx context.Context, runtimeFrameworkID, repoID int64, deployType int) error {
+ relation := RepositoriesRuntimeFramework{
+ RuntimeFrameworkID: runtimeFrameworkID,
+ RepoID: repoID,
+ Type: deployType,
+ }
+ _, err := m.db.Operator.Core.NewInsert().Model(&relation).Exec(ctx)
+ return err
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) Delete(ctx context.Context, runtimeFrameworkID, repoID int64, deployType int) error {
+ res, err := m.db.BunDB.Exec("delete from repositories_runtime_frameworks where type = ? and repo_id = ? and runtime_framework_id = ?", deployType, repoID, runtimeFrameworkID)
+ if err != nil {
+ return err
+ }
+ err = assertAffectedOneRow(res, err)
+ return err
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) DeleteByRepoID(ctx context.Context, repoID int64) error {
+ _, err := m.db.Operator.Core.NewDelete().Model((*RepositoriesRuntimeFramework)(nil)).Where("repo_id = ?", repoID).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("delete repo runtime failed, %w", err)
+ }
+ return nil
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) GetByIDsAndType(ctx context.Context, runtimeFrameworkID, repoID int64, deployType int) ([]RepositoriesRuntimeFramework, error) {
+ var result []RepositoriesRuntimeFramework
+ _, err := m.db.Operator.Core.NewSelect().Model(&result).Where("type = ? and repo_id=? and runtime_framework_id = ?", deployType, repoID, runtimeFrameworkID).Exec(ctx, &result)
+ return result, err
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) ListRepoIDsByType(ctx context.Context, deployType int) ([]RepositoriesRuntimeFramework, error) {
+ var result []RepositoriesRuntimeFramework
+ _, err := m.db.Operator.Core.NewSelect().Model(&result).Where("type = ?", deployType).Exec(ctx, &result)
+ return result, err
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) GetByRepoIDsAndType(ctx context.Context, repoID int64, deployType int) ([]RepositoriesRuntimeFramework, error) {
+ var result []RepositoriesRuntimeFramework
+ _, err := m.db.Operator.Core.NewSelect().Model(&result).Where("type = ? and repo_id=?", deployType, repoID).Exec(ctx, &result)
+ return result, err
+}
+
+func (m *repositoriesRuntimeFrameworkStoreImpl) GetByRepoIDs(ctx context.Context, repoID int64) ([]RepositoriesRuntimeFramework, error) {
+ var result []RepositoriesRuntimeFramework
+ _, err := m.db.Operator.Core.NewSelect().Model(&result).Where("repo_id=?", repoID).Exec(ctx, &result)
+ if err != nil {
+ return nil, fmt.Errorf("get runtime by repoid failed, %w", err)
+ }
+ return result, nil
+}
+
+
+
package database
+
+import (
+ "context"
+)
+
+type resourceModelStoreImpl struct {
+ db *DB
+}
+
+type ResourceModelStore interface {
+ // find multi Resource model by model name with fuzzy matching, parameter modelName like model_name in db
+ FindByModelName(ctx context.Context, modelName string) ([]*ResourceModel, error)
+ // find model by name which is in resource model table but not in runtime framework repo
+ CheckModelNameNotInRFRepo(ctx context.Context, modelName string, repoId int64) (*ResourceModel, error)
+}
+
+func NewResourceModelStore() ResourceModelStore {
+ return &resourceModelStoreImpl{db: defaultDB}
+}
+
+func NewResourceModelStoreWithDB(db *DB) ResourceModelStore {
+ return &resourceModelStoreImpl{db: db}
+}
+
+type ResourceModel struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ ResourceName string `bun:",notnull" json:"resource_name"`
+ EngineName string `bun:",notnull" json:"engine_name"`
+ ModelName string `bun:",notnull" json:"model_name"`
+ Type string `bun:",notnull" json:"type"`
+ times
+}
+
+// find multi Resource model by model name with fuzzy matching, parameter modelName like model_name in db
+func (s *resourceModelStoreImpl) FindByModelName(ctx context.Context, modelName string) ([]*ResourceModel, error) {
+ var models []*ResourceModel
+ err := s.db.Core.NewSelect().Model(&models).Where("model_name LIKE ?", "%"+modelName+"%").Scan(ctx)
+ return models, err
+}
+
+// find model by name which is in resource model table but not in runtime framework repo
+func (s *resourceModelStoreImpl) CheckModelNameNotInRFRepo(ctx context.Context, modelName string, repoId int64) (*ResourceModel, error) {
+ var rm ResourceModel
+ _, err := s.db.Core.NewSelect().Model(&rm).
+ Where("LOWER(model_name) LIKE ?", "%"+modelName+"%").
+ Exec(ctx, &rm)
+ if err != nil {
+ return nil, err
+ }
+
+ var rrfs []*RepositoriesRuntimeFramework
+ err = s.db.Core.NewSelect().Model(&rrfs).Where("repo_id = ?", repoId).
+ Scan(ctx)
+ if err != nil || len(rrfs) > 0 {
+ return nil, err
+ }
+
+ return &rm, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "strings"
+)
+
+type runtimeArchitecturesStoreImpl struct {
+ db *DB
+}
+
+type RuntimeArchitecturesStore interface {
+ ListByRuntimeFrameworkID(ctx context.Context, id int64) ([]RuntimeArchitecture, error)
+ Add(ctx context.Context, arch RuntimeArchitecture) error
+ DeleteByRuntimeIDAndArchName(ctx context.Context, id int64, archName string) error
+ FindByRuntimeIDAndArchName(ctx context.Context, id int64, archName string) (*RuntimeArchitecture, error)
+ ListByRArchName(ctx context.Context, archName string) ([]RuntimeArchitecture, error)
+ ListByRArchNameAndModel(ctx context.Context, archName, modelName string) ([]RuntimeArchitecture, error)
+ GetRuntimeByModelName(ctx context.Context, archName, modelName string) ([]RuntimeArchitecture, error)
+}
+
+func NewRuntimeArchitecturesStore() RuntimeArchitecturesStore {
+ return &runtimeArchitecturesStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRuntimeArchitecturesStoreWithDB(db *DB) RuntimeArchitecturesStore {
+ return &runtimeArchitecturesStoreImpl{
+ db: db,
+ }
+}
+
+type RuntimeArchitecture struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ RuntimeFrameworkID int64 `bun:",notnull" json:"runtime_framework_id"`
+ ArchitectureName string `bun:",notnull" json:"architecture_name"`
+}
+
+func (ra *runtimeArchitecturesStoreImpl) ListByRuntimeFrameworkID(ctx context.Context, id int64) ([]RuntimeArchitecture, error) {
+ var result []RuntimeArchitecture
+ _, err := ra.db.Operator.Core.NewSelect().Model(&result).Where("runtime_framework_id = ?", id).Exec(ctx, &result)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting runtime architecture, %w", err)
+ }
+ return result, nil
+}
+
+func (ra *runtimeArchitecturesStoreImpl) Add(ctx context.Context, arch RuntimeArchitecture) error {
+ res, err := ra.db.Core.NewInsert().Model(&arch).Exec(ctx, &arch)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("creating runtime architecture in the db failed,error:%w", err)
+ }
+ return nil
+}
+
+func (ra *runtimeArchitecturesStoreImpl) DeleteByRuntimeIDAndArchName(ctx context.Context, id int64, archName string) error {
+ var arch RuntimeArchitecture
+ _, err := ra.db.Core.NewDelete().Model(&arch).Where("runtime_framework_id = ? and architecture_name = ?", id, archName).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("deleting runtime architecture in the db failed, error:%w", err)
+ }
+ return nil
+}
+
+func (ra *runtimeArchitecturesStoreImpl) FindByRuntimeIDAndArchName(ctx context.Context, id int64, archName string) (*RuntimeArchitecture, error) {
+ var arch RuntimeArchitecture
+ _, err := ra.db.Core.NewSelect().Model(&arch).Where("runtime_framework_id = ? and architecture_name = ?", id, archName).Exec(ctx, &arch)
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ if err != nil {
+ return nil, fmt.Errorf("getting runtime architecture in the db failed, error:%w", err)
+ }
+ return &arch, nil
+}
+
+func (ra *runtimeArchitecturesStoreImpl) ListByRArchName(ctx context.Context, archName string) ([]RuntimeArchitecture, error) {
+ var result []RuntimeArchitecture
+ _, err := ra.db.Operator.Core.NewSelect().Model(&result).Where("architecture_name = ?", archName).Exec(ctx, &result)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting runtime architecture, %w", err)
+ }
+ return result, nil
+}
+
+func (ra *runtimeArchitecturesStoreImpl) ListByRArchNameAndModel(ctx context.Context, archName, modelName string) ([]RuntimeArchitecture, error) {
+ var result []RuntimeArchitecture
+ _, err := ra.db.Operator.Core.NewSelect().Model(&result).Where("architecture_name = ?", archName).Exec(ctx, &result)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting runtime architecture, %w", err)
+ }
+ result2, err := ra.GetRuntimeByModelName(ctx, archName, modelName)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting runtime architecture by model name, %w", err)
+ }
+ result = append(result, result2...)
+ return result, nil
+}
+
+/**
+@Description: get runtime architecture by model name
+@param ctx
+@param archName
+@param modelName
+@return []RuntimeArchitecture
+------------ table resource_models -----------------
+resource_name | engine_name | model_name
+---------------+-------------+-----------------------------------
+ascend | mindie | Baichuan-7B
+ascend | mindie | Baichuan-13B
+nvidia | nim | Llama-3.1-8B-Instruct
+nvidia | nim | Llama-3-8B-Instruct
+
+------------ table runtime_frameworks -----------------
+ frame_name | frame_image | frame_npu_image
+---------------------------+----------------------------------------------------------------+------------------------
+ TGI | tgi:2.1 |
+ VLLM | vllm-local:2.7 | vllm-cpu:2.3
+ MindIE | | mindie:1.8-csg-1.0.RC2
+ nim-llama-3.1-8b-instruct | nvcr.io/nim/meta/llama-3.1-8b-instruct:latest |
+ nim-llama-2-13b-chat | nvcr.io/nim/meta/llama-2-13b-chat:latest |
+ nim-llama3-8b-instruct | nvcr.io/nim/meta/llama3-8b-instruct:latest |
+case 1: mindie
+all models share same runtime framework mindie, so we only need to get the runtime framework for mindie when the image contains mindie
+case 2: nim
+every llama model has its own runtime framework, so we need to get the runtime framework for each model
+Meta-Llama-3-8B-Instruct --> llama3-8b-instruct
+Llama-2-13b-chat --> llama-2-13b-chat
+*/
+
+func (ra *runtimeArchitecturesStoreImpl) GetRuntimeByModelName(ctx context.Context, archName, modelName string) ([]RuntimeArchitecture, error) {
+ var result []RuntimeArchitecture
+ var resModel []ResourceModel
+ err := ra.db.Core.NewSelect().Model(&resModel).Where("LOWER(model_name) like ? and engine_name != ?", fmt.Sprintf("%%%s%%", strings.ToLower(modelName)), "nim").Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting resource model, %w", err)
+ }
+ var resNIMModel []ResourceModel
+ nimModel := strings.Replace(strings.ToLower(modelName), "meta-", "", 1)
+ err = ra.db.Core.NewSelect().Model(&resNIMModel).Where("LOWER(model_name) = ? and engine_name = ?", nimModel, "nim").Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting resource model, %w", err)
+ }
+ resModel = append(resModel, resNIMModel...)
+ var runtime_frameworks []RuntimeFramework
+ // select all runtime_frameworks
+ err = ra.db.Core.NewSelect().Model(&runtime_frameworks).Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("error happened while getting runtime frameworks, %w", err)
+ }
+ nimMatchModel := strings.ReplaceAll(nimModel, "-", "")
+ for _, r := range resModel {
+ //append to result if result dont' have the runtime_framework_id
+ engineName := strings.ToLower(r.EngineName)
+
+ for _, rf := range runtime_frameworks {
+ image := strings.ToLower(rf.FrameImage)
+ if strings.Contains(image, "/") {
+ parts := strings.Split(image, "/")
+ image = parts[len(parts)-1]
+ }
+ if (strings.Contains(image, engineName) || strings.Contains(rf.FrameNpuImage, engineName)) && !contains(result, rf.ID) {
+ result = append(result, RuntimeArchitecture{
+ RuntimeFrameworkID: rf.ID,
+ ArchitectureName: archName,
+ })
+ continue
+ }
+ // special handling for nim models
+ nimImage := strings.ReplaceAll(image, "-", "")
+ if strings.Contains(nimImage, nimMatchModel) && !contains(result, rf.ID) {
+ result = append(result, RuntimeArchitecture{
+ RuntimeFrameworkID: rf.ID,
+ ArchitectureName: archName,
+ })
+ }
+ }
+
+ }
+ return result, nil
+}
+
+func contains(architectures []RuntimeArchitecture, id int64) bool {
+ for _, arch := range architectures {
+ if arch.RuntimeFrameworkID == id {
+ return true
+ }
+ }
+ return false
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/uptrace/bun"
+)
+
+type runtimeFrameworksStoreImpl struct {
+ db *DB
+}
+
+type RuntimeFrameworksStore interface {
+ List(ctx context.Context, deployType int) ([]RuntimeFramework, error)
+ ListByRepoID(ctx context.Context, repoID int64, deployType int) ([]RepositoriesRuntimeFramework, error)
+ FindByID(ctx context.Context, id int64) (*RuntimeFramework, error)
+ Add(ctx context.Context, frame RuntimeFramework) error
+ Update(ctx context.Context, frame RuntimeFramework) (*RuntimeFramework, error)
+ Delete(ctx context.Context, frame RuntimeFramework) error
+ FindEnabledByID(ctx context.Context, id int64) (*RuntimeFramework, error)
+ FindEnabledByName(ctx context.Context, name string) (*RuntimeFramework, error)
+ ListAll(ctx context.Context) ([]RuntimeFramework, error)
+ ListByIDs(ctx context.Context, ids []int64) ([]RuntimeFramework, error)
+}
+
+func NewRuntimeFrameworksStore() RuntimeFrameworksStore {
+ return &runtimeFrameworksStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewRuntimeFrameworksStoreWithDB(db *DB) RuntimeFrameworksStore {
+ return &runtimeFrameworksStoreImpl{
+ db: db,
+ }
+}
+
+type RuntimeFramework struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ FrameName string `bun:",notnull" json:"frame_name"`
+ FrameVersion string `bun:",notnull" json:"frame_version"`
+ FrameImage string `bun:",notnull" json:"frame_image"`
+ FrameNpuImage string `bun:",notnull" json:"frame_npu_image"`
+ FrameCpuImage string `bun:",notnull" json:"frame_cpu_image"`
+ Enabled int64 `bun:",notnull" json:"enabled"`
+ ContainerPort int `bun:",notnull" json:"container_port"`
+ Type int `bun:",notnull" json:"type"` // 0-space, 1-inference, 2-finetune
+ times
+}
+
+func (rf *runtimeFrameworksStoreImpl) List(ctx context.Context, deployType int) ([]RuntimeFramework, error) {
+ var result []RuntimeFramework
+ _, err := rf.db.Operator.Core.NewSelect().Model(&result).Where("type = ?", deployType).Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func (rf *runtimeFrameworksStoreImpl) ListByRepoID(ctx context.Context, repoID int64, deployType int) ([]RepositoriesRuntimeFramework, error) {
+ var result []RepositoriesRuntimeFramework
+ err := rf.db.Operator.Core.NewSelect().Model(&RepositoriesRuntimeFramework{}).Relation("RuntimeFramework").Where("repositories_runtime_framework.type = ? and repositories_runtime_framework.repo_id = ?", deployType, repoID).Scan(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, err
+}
+
+func (rf *runtimeFrameworksStoreImpl) FindByID(ctx context.Context, id int64) (*RuntimeFramework, error) {
+ var res RuntimeFramework
+ res.ID = id
+ _, err := rf.db.Core.NewSelect().Model(&res).WherePK().Exec(ctx, &res)
+ return &res, err
+}
+
+func (rf *runtimeFrameworksStoreImpl) Add(ctx context.Context, frame RuntimeFramework) error {
+ res, err := rf.db.Core.NewInsert().Model(&frame).Exec(ctx, &frame)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create runtime framework in db failed", slog.String("error", err.Error()))
+ return fmt.Errorf("create runtime framework in db failed,error:%w", err)
+ }
+ return nil
+}
+
+func (rf *runtimeFrameworksStoreImpl) Update(ctx context.Context, frame RuntimeFramework) (*RuntimeFramework, error) {
+ _, err := rf.db.Core.NewUpdate().Model(&frame).WherePK().Exec(ctx)
+ return &frame, err
+}
+
+func (rf *runtimeFrameworksStoreImpl) Delete(ctx context.Context, frame RuntimeFramework) error {
+ _, err := rf.db.Core.NewDelete().Model(&frame).WherePK().Exec(ctx)
+ return err
+}
+
+func (rf *runtimeFrameworksStoreImpl) FindEnabledByID(ctx context.Context, id int64) (*RuntimeFramework, error) {
+ var res RuntimeFramework
+ res.ID = id
+ _, err := rf.db.Core.NewSelect().Model(&res).WherePK().Where("enabled = 1").Exec(ctx, &res)
+ return &res, err
+}
+
+func (rf *runtimeFrameworksStoreImpl) FindEnabledByName(ctx context.Context, name string) (*RuntimeFramework, error) {
+ var res RuntimeFramework
+ _, err := rf.db.Core.NewSelect().Model(&res).Where("frame_name = ?", name).Where("enabled = 1").Exec(ctx, &res)
+ return &res, err
+}
+
+func (rf *runtimeFrameworksStoreImpl) ListAll(ctx context.Context) ([]RuntimeFramework, error) {
+ var result []RuntimeFramework
+ _, err := rf.db.Operator.Core.NewSelect().Model(&result).Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func (rf *runtimeFrameworksStoreImpl) ListByIDs(ctx context.Context, ids []int64) ([]RuntimeFramework, error) {
+ var result []RuntimeFramework
+ _, err := rf.db.Operator.Core.NewSelect().Model(&result).Where("id in (?)", bun.In(ids)).Exec(ctx, &result)
+ if err != nil {
+ return nil, fmt.Errorf("query runtimes failed, %w", err)
+ }
+ return result, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type spaceStoreImpl struct {
+ db *DB
+}
+
+type SpaceStore interface {
+ // BeginTx(ctx context.Context) (bun.Tx, error)
+ // CreateTx(ctx context.Context, tx bun.Tx, input Space) (*Space, error)
+ Create(ctx context.Context, input Space) (*Space, error)
+ Update(ctx context.Context, input Space) (err error)
+ FindByPath(ctx context.Context, namespace, name string) (*Space, error)
+ Delete(ctx context.Context, input Space) error
+ ByID(ctx context.Context, id int64) (*Space, error)
+ // ByRepoIDs get spaces by repoIDs, only basice info, no related repo
+ ByRepoIDs(ctx context.Context, repoIDs []int64) (spaces []Space, err error)
+ ByRepoID(ctx context.Context, repoID int64) (*Space, error)
+ ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (spaces []Space, total int, err error)
+ ByUserLikes(ctx context.Context, userID int64, per, page int) (spaces []Space, total int, err error)
+ ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (spaces []Space, total int, err error)
+ ListByPath(ctx context.Context, paths []string) ([]Space, error)
+}
+
+func NewSpaceStore() SpaceStore {
+ return &spaceStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewSpaceStoreWithDB(db *DB) SpaceStore {
+ return &spaceStoreImpl{
+ db: db,
+ }
+}
+
+// func (s *spaceStoreImpl) BeginTx(ctx context.Context) (bun.Tx, error) {
+// return s.db.Core.BeginTx(ctx, nil)
+// }
+
+// func (s *spaceStoreImpl) CreateTx(ctx context.Context, tx bun.Tx, input Space) (*Space, error) {
+// res, err := tx.NewInsert().Model(&input).Exec(ctx)
+// if err := assertAffectedOneRow(res, err); err != nil {
+// slog.Error("create space in tx failed", slog.String("error", err.Error()))
+// return nil, fmt.Errorf("create space in tx failed,error:%w", err)
+// }
+
+// input.ID, _ = res.LastInsertId()
+// return &input, nil
+// }
+
+func (s *spaceStoreImpl) Create(ctx context.Context, input Space) (*Space, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ slog.Error("create space in db failed", slog.String("error", err.Error()))
+ return nil, fmt.Errorf("create space in db failed,error:%w", err)
+ }
+
+ input.ID, _ = res.LastInsertId()
+ return &input, nil
+}
+
+func (s *spaceStoreImpl) Update(ctx context.Context, input Space) (err error) {
+ _, err = s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+ return
+}
+
+func (s *spaceStoreImpl) FindByPath(ctx context.Context, namespace, name string) (*Space, error) {
+ resSpace := new(Space)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(resSpace).
+ Relation("Repository.User").
+ Where("repository.path = ? and repository.repository_type = ?", fmt.Sprintf("%s/%s", namespace, name), types.SpaceRepo).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find space: %w", err)
+ }
+
+ return resSpace, err
+}
+
+func (s *spaceStoreImpl) Delete(ctx context.Context, input Space) error {
+ res, err := s.db.Operator.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("delete space in tx failed,error:%w", err)
+ }
+ return nil
+}
+
+func (s *spaceStoreImpl) ByID(ctx context.Context, id int64) (*Space, error) {
+ var space Space
+ err := s.db.Core.NewSelect().Model(&space).Relation("Repository").Where("space.id = ?", id).Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &space, err
+}
+
+// ByRepoIDs get spaces by repoIDs, only basice info, no related repo
+func (s *spaceStoreImpl) ByRepoIDs(ctx context.Context, repoIDs []int64) (spaces []Space, err error) {
+ err = s.db.Operator.Core.NewSelect().
+ Model(&spaces).
+ Where("repository_id in (?)", bun.In(repoIDs)).
+ Scan(ctx)
+
+ return
+}
+
+func (s *spaceStoreImpl) ByRepoID(ctx context.Context, repoID int64) (*Space, error) {
+ var space Space
+ err := s.db.Core.NewSelect().Model(&space).Where("repository_id = ?", repoID).Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find space by id, repository id: %d,error: %w", repoID, err)
+ }
+ return &space, err
+}
+
+func (s *spaceStoreImpl) ByUsername(ctx context.Context, username string, per, page int, onlyPublic bool) (spaces []Space, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&spaces).
+ Relation("Repository.Tags").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", username))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("space.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *spaceStoreImpl) ByUserLikes(ctx context.Context, userID int64, per, page int) (spaces []Space, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&spaces).
+ Relation("Repository.Tags").
+ Where("repository.id in (select repo_id from user_likes where user_id=?)", userID)
+
+ query = query.Order("space.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *spaceStoreImpl) ByOrgPath(ctx context.Context, namespace string, per, page int, onlyPublic bool) (spaces []Space, total int, err error) {
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&spaces).
+ Relation("Repository.Tags").
+ Relation("Repository.User").
+ Where("repository.path like ?", fmt.Sprintf("%s/%%", namespace))
+
+ if onlyPublic {
+ query = query.Where("repository.private = ?", false)
+ }
+ query = query.Order("space.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx, &spaces)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *spaceStoreImpl) ListByPath(ctx context.Context, paths []string) ([]Space, error) {
+ var spaces []Space
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&Space{}).
+ Relation("Repository").
+ Where("path IN (?)", bun.In(paths)).
+ Scan(ctx, &spaces)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find space by path,error: %w", err)
+ }
+
+ var sortedSpaces []Space
+ for _, path := range paths {
+ for _, ds := range spaces {
+ if ds.Repository.Path == path {
+ sortedSpaces = append(sortedSpaces, ds)
+ }
+ }
+ }
+
+ spaces = nil
+ return sortedSpaces, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type spaceResourceStoreImpl struct {
+ db *DB
+}
+
+type SpaceResourceStore interface {
+ Index(ctx context.Context, clusterId string) ([]SpaceResource, error)
+ Create(ctx context.Context, input SpaceResource) (*SpaceResource, error)
+ Update(ctx context.Context, input SpaceResource) (*SpaceResource, error)
+ Delete(ctx context.Context, input SpaceResource) error
+ FindByID(ctx context.Context, id int64) (*SpaceResource, error)
+ FindByName(ctx context.Context, name string) (*SpaceResource, error)
+ FindAll(ctx context.Context) ([]SpaceResource, error)
+}
+
+func NewSpaceResourceStore() SpaceResourceStore {
+ return &spaceResourceStoreImpl{db: defaultDB}
+}
+
+func NewSpaceResourceStoreWithDB(db *DB) SpaceResourceStore {
+ return &spaceResourceStoreImpl{db: db}
+}
+
+type SpaceResource struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name"`
+ Resources string `bun:",notnull" json:"resources"`
+ ClusterID string `bun:",notnull" json:"cluster_id"`
+ times
+}
+
+func (s *spaceResourceStoreImpl) Index(ctx context.Context, clusterId string) ([]SpaceResource, error) {
+ var result []SpaceResource
+ _, err := s.db.Operator.Core.NewSelect().Model(&result).Where("cluster_id = ?", clusterId).Order("name asc").Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func (s *spaceResourceStoreImpl) Create(ctx context.Context, input SpaceResource) (*SpaceResource, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create space resource in tx failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *spaceResourceStoreImpl) Update(ctx context.Context, input SpaceResource) (*SpaceResource, error) {
+ _, err := s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+
+ return &input, err
+}
+
+func (s *spaceResourceStoreImpl) Delete(ctx context.Context, input SpaceResource) error {
+ _, err := s.db.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+
+ return err
+}
+
+func (s *spaceResourceStoreImpl) FindByID(ctx context.Context, id int64) (*SpaceResource, error) {
+ var res SpaceResource
+ res.ID = id
+ _, err := s.db.Core.NewSelect().Model(&res).WherePK().Exec(ctx, &res)
+
+ return &res, err
+}
+
+func (s *spaceResourceStoreImpl) FindByName(ctx context.Context, name string) (*SpaceResource, error) {
+ var res SpaceResource
+ err := s.db.Core.NewSelect().Model(&res).Where("name = ?", name).Scan(ctx)
+
+ return &res, err
+}
+
+func (s *spaceResourceStoreImpl) FindAll(ctx context.Context) ([]SpaceResource, error) {
+ var result []SpaceResource
+ _, err := s.db.Operator.Core.NewSelect().Model(&result).Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+)
+
+type spaceSdkStoreImpl struct {
+ db *DB
+}
+
+type SpaceSdkStore interface {
+ Index(ctx context.Context) ([]SpaceSdk, error)
+ Create(ctx context.Context, input SpaceSdk) (*SpaceSdk, error)
+ Update(ctx context.Context, input SpaceSdk) (*SpaceSdk, error)
+ Delete(ctx context.Context, input SpaceSdk) error
+ FindByID(ctx context.Context, id int64) (*SpaceSdk, error)
+}
+
+func NewSpaceSdkStore() SpaceSdkStore {
+ return &spaceSdkStoreImpl{db: defaultDB}
+}
+
+func NewSpaceSdkStoreWithDB(db *DB) SpaceSdkStore {
+ return &spaceSdkStoreImpl{db: db}
+}
+
+type SpaceSdk struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name"`
+ Version string `bun:",notnull" json:"version"`
+ times
+}
+
+func (s *spaceSdkStoreImpl) Index(ctx context.Context) ([]SpaceSdk, error) {
+ var result []SpaceSdk
+ _, err := s.db.Operator.Core.
+ NewSelect().
+ Model(&result).
+ Exec(ctx, &result)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+}
+
+func (s *spaceSdkStoreImpl) Create(ctx context.Context, input SpaceSdk) (*SpaceSdk, error) {
+ res, err := s.db.Core.NewInsert().Model(&input).Exec(ctx, &input)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return nil, fmt.Errorf("create space sdk in tx failed,error:%w", err)
+ }
+
+ return &input, nil
+}
+
+func (s *spaceSdkStoreImpl) Update(ctx context.Context, input SpaceSdk) (*SpaceSdk, error) {
+ _, err := s.db.Core.NewUpdate().Model(&input).WherePK().Exec(ctx)
+
+ return &input, err
+}
+
+func (s *spaceSdkStoreImpl) Delete(ctx context.Context, input SpaceSdk) error {
+ _, err := s.db.Core.NewDelete().Model(&input).WherePK().Exec(ctx)
+
+ return err
+}
+
+func (s *spaceSdkStoreImpl) FindByID(ctx context.Context, id int64) (*SpaceSdk, error) {
+ var res SpaceSdk
+ res.ID = id
+ _, err := s.db.Core.NewSelect().Model(&res).WherePK().Exec(ctx, &res)
+
+ return &res, err
+}
+
+
+
package database
+
+import (
+ "context"
+)
+
+type sSHKeyStoreImpl struct {
+ db *DB
+}
+
+type SSHKeyStore interface {
+ Index(ctx context.Context, username string, per, page int) (sshKeys []SSHKey, err error)
+ Create(ctx context.Context, sshKey *SSHKey) (*SSHKey, error)
+ FindByID(ctx context.Context, id int64) (*SSHKey, error)
+ FindByFingerpringSHA256(ctx context.Context, fingerprint string) (*SSHKey, error)
+ Delete(ctx context.Context, id int64) (err error)
+ IsExist(ctx context.Context, username, keyName string) (exists bool, err error)
+ FindByUsernameAndName(ctx context.Context, username, keyName string) (sshKey SSHKey, err error)
+ FindByKeyContent(ctx context.Context, key string) (*SSHKey, error)
+ FindByNameAndUserID(ctx context.Context, name string, userID int64) (*SSHKey, error)
+}
+
+func NewSSHKeyStore() SSHKeyStore {
+ return &sSHKeyStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewSSHKeyStoreWithDB(db *DB) SSHKeyStore {
+ return &sSHKeyStoreImpl{
+ db: db,
+ }
+}
+
+type SSHKey struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ GitID int64 `bun:",notnull" json:"git_id"`
+ Name string `bun:",notnull" json:"name"`
+ Content string `bun:",notnull" json:"content"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ User *User `bun:"rel:belongs-to,join:user_id=id" json:"user"`
+ FingerprintSHA256 string `bun:"," json:"fingerprint_sha256"`
+ times
+}
+
+func (s *sSHKeyStoreImpl) Index(ctx context.Context, username string, per, page int) (sshKeys []SSHKey, err error) {
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(&sshKeys).
+ Relation("User").
+ Join("JOIN users AS u ON u.id = ssh_key.user_id").
+ Where("u.username = ?", username).
+ Order("created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per).
+ Scan(ctx)
+ return
+}
+
+func (s *sSHKeyStoreImpl) Create(ctx context.Context, sshKey *SSHKey) (*SSHKey, error) {
+ err := s.db.Operator.Core.
+ NewInsert().
+ Model(sshKey).
+ Returning("*").
+ Scan(ctx)
+ return sshKey, err
+}
+
+func (s *sSHKeyStoreImpl) FindByID(ctx context.Context, id int64) (*SSHKey, error) {
+ var sshKey SSHKey
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&sshKey).
+ Relation("User").
+ Where("ssh_key.id = ?", id).
+ Scan(ctx)
+ return &sshKey, err
+}
+
+func (s *sSHKeyStoreImpl) FindByFingerpringSHA256(ctx context.Context, fingerprint string) (*SSHKey, error) {
+ var sshKey SSHKey
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(&sshKey).
+ Relation("User").
+ Where("ssh_key.fingerprint_sha256 = ?", fingerprint).
+ Scan(ctx)
+ return &sshKey, err
+}
+
+func (s *sSHKeyStoreImpl) Delete(ctx context.Context, id int64) (err error) {
+ var sshKey SSHKey
+ _, err = s.db.Operator.Core.
+ NewDelete().
+ Model(&sshKey).
+ Where("id = ?", id).
+ Exec(ctx)
+ return
+}
+
+func (s *sSHKeyStoreImpl) IsExist(ctx context.Context, username, keyName string) (exists bool, err error) {
+ var sshKey SSHKey
+ exists, err = s.db.Operator.Core.
+ NewSelect().
+ Model(&sshKey).
+ Join("JOIN users AS u ON u.id = ssh_key.user_id").
+ Where("u.username = ?", username).
+ Where("ssh_key.name = ?", keyName).
+ Exists(ctx)
+ return
+}
+
+func (s *sSHKeyStoreImpl) FindByUsernameAndName(ctx context.Context, username, keyName string) (sshKey SSHKey, err error) {
+ sshKey.Name = keyName
+ err = s.db.Operator.Core.
+ NewSelect().
+ Model(&sshKey).
+ Join("JOIN users AS u ON u.id = ssh_key.user_id").
+ Where("u.username = ?", username).
+ Where("ssh_key.name = ?", keyName).
+ Scan(ctx)
+ return sshKey, err
+}
+
+func (s *sSHKeyStoreImpl) FindByKeyContent(ctx context.Context, key string) (*SSHKey, error) {
+ sshKey := new(SSHKey)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(sshKey).
+ Where("content = ?", key).
+ Scan(ctx)
+ return sshKey, err
+}
+
+func (s *sSHKeyStoreImpl) FindByNameAndUserID(ctx context.Context, name string, userID int64) (*SSHKey, error) {
+ sshKey := new(SSHKey)
+ err := s.db.Operator.Core.
+ NewSelect().
+ Model(sshKey).
+ Where("name = ? and user_id = ?", name, userID).
+ Scan(ctx)
+ return sshKey, err
+}
+
+
+
package database
+
+import "context"
+
+type syncClientSettingStoreImpl struct {
+ db *DB
+}
+
+type SyncClientSettingStore interface {
+ Create(ctx context.Context, setting *SyncClientSetting) (*SyncClientSetting, error)
+ SyncClientSettingExists(ctx context.Context) (bool, error)
+ DeleteAll(ctx context.Context) error
+ First(ctx context.Context) (*SyncClientSetting, error)
+}
+
+func NewSyncClientSettingStore() SyncClientSettingStore {
+ return &syncClientSettingStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewSyncClientSettingStoreWithDB(db *DB) SyncClientSettingStore {
+ return &syncClientSettingStoreImpl{
+ db: db,
+ }
+}
+
+type SyncClientSetting struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Token string `bun:",notnull" json:"token"`
+ ConcurrentCount int `bun:",nullzero" json:"concurrent_count"`
+ MaxBandwidth int `bun:",nullzero" json:"max_bandwidth"`
+ IsDefault bool `bun:"," json:"default"`
+ times
+}
+
+func (s *syncClientSettingStoreImpl) Create(ctx context.Context, setting *SyncClientSetting) (*SyncClientSetting, error) {
+ err := s.db.Operator.Core.NewInsert().
+ Model(setting).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return setting, nil
+}
+
+func (s *syncClientSettingStoreImpl) SyncClientSettingExists(ctx context.Context) (bool, error) {
+ return s.db.Operator.Core.NewSelect().
+ Model((*SyncClientSetting)(nil)).
+ Exists(ctx)
+}
+
+func (s *syncClientSettingStoreImpl) DeleteAll(ctx context.Context) error {
+ _, err := s.db.Operator.Core.NewDelete().Model((*SyncClientSetting)(nil)).Where("1=1").Exec(ctx)
+ return err
+}
+
+func (s *syncClientSettingStoreImpl) First(ctx context.Context) (*SyncClientSetting, error) {
+ var mt SyncClientSetting
+ err := s.db.Operator.Core.NewSelect().
+ Model(&mt).
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &mt, nil
+}
+
+
+
package database
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type syncVersionStoreImpl struct {
+ db *DB
+}
+
+type SyncVersionSource int
+
+type SyncVersionStore interface {
+ Create(ctx context.Context, version *SyncVersion) (err error)
+ BatchCreate(ctx context.Context, versions []SyncVersion) error
+ FindByPath(ctx context.Context, path string) (*SyncVersion, error)
+ FindByRepoTypeAndPath(ctx context.Context, path string, repoType types.RepositoryType) (*SyncVersion, error)
+}
+
+func NewSyncVersionStore() SyncVersionStore {
+ return &syncVersionStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewSyncVersionStoreWithDB(db *DB) SyncVersionStore {
+ return &syncVersionStoreImpl{
+ db: db,
+ }
+}
+
+func (s *syncVersionStoreImpl) Create(ctx context.Context, version *SyncVersion) (err error) {
+ _, err = s.db.Operator.Core.NewInsert().Model(version).Exec(ctx)
+ return
+}
+
+func (s *syncVersionStoreImpl) BatchCreate(ctx context.Context, versions []SyncVersion) error {
+ result, err := s.db.Core.NewInsert().Model(&versions).Exec(ctx)
+ return assertAffectedXRows(int64(len(versions)), result, err)
+}
+
+func (s *syncVersionStoreImpl) FindByPath(ctx context.Context, path string) (*SyncVersion, error) {
+ var syncVersion SyncVersion
+ err := s.db.Core.NewSelect().
+ Model(&syncVersion).
+ Where("repo_path = ?", path).
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &syncVersion, nil
+}
+
+func (s *syncVersionStoreImpl) FindByRepoTypeAndPath(ctx context.Context, path string, repoType types.RepositoryType) (*SyncVersion, error) {
+ var syncVersion SyncVersion
+ err := s.db.Core.NewSelect().
+ Model(&syncVersion).
+ Where("repo_path = ? and repo_type = ?", path, repoType).
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &syncVersion, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "github.com/uptrace/bun"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type tagStoreImpl struct {
+ db *DB
+}
+
+type TagStore interface {
+ // Alltags returns all tags in the database
+ AllTags(ctx context.Context) ([]Tag, error)
+ AllTagsByScope(ctx context.Context, scope TagScope) ([]*Tag, error)
+ AllTagsByScopeAndCategory(ctx context.Context, scope TagScope, category string) ([]*Tag, error)
+ GetTagsByScopeAndCategories(ctx context.Context, scope TagScope, categories []string) ([]*Tag, error)
+ AllModelTags(ctx context.Context) ([]*Tag, error)
+ AllPromptTags(ctx context.Context) ([]*Tag, error)
+ AllDatasetTags(ctx context.Context) ([]*Tag, error)
+ AllCodeTags(ctx context.Context) ([]*Tag, error)
+ AllSpaceTags(ctx context.Context) ([]*Tag, error)
+ AllModelCategories(ctx context.Context) ([]TagCategory, error)
+ AllPromptCategories(ctx context.Context) ([]TagCategory, error)
+ AllDatasetCategories(ctx context.Context) ([]TagCategory, error)
+ AllCodeCategories(ctx context.Context) ([]TagCategory, error)
+ AllSpaceCategories(ctx context.Context) ([]TagCategory, error)
+ CreateTag(ctx context.Context, category, name, group string, scope TagScope) (Tag, error)
+ SaveTags(ctx context.Context, tags []*Tag) error
+ // SetMetaTags will delete existing tags and create new ones
+ SetMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string, tags []*Tag) (repoTags []*RepositoryTag, err error)
+ SetLibraryTag(ctx context.Context, repoType types.RepositoryType, namespace, name string, newTag, oldTag *Tag) (err error)
+ UpsertRepoTags(ctx context.Context, repoID int64, oldTagIDs, newTagIDs []int64) (err error)
+ RemoveRepoTags(ctx context.Context, repoID int64, tagIDs []int64) (err error)
+ FindOrCreate(ctx context.Context, tag Tag) (*Tag, error)
+ FindTag(ctx context.Context, name, scope, category string) (*Tag, error)
+}
+
+func NewTagStore() TagStore {
+ return &tagStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewTagStoreWithDB(db *DB) TagStore {
+ return &tagStoreImpl{
+ db: db,
+ }
+}
+
+type TagScope string
+
+const (
+ ModelTagScope TagScope = "model"
+ DatasetTagScope TagScope = "dataset"
+ CodeTagScope TagScope = "code"
+ SpaceTagScope TagScope = "space"
+ PromptTagScope TagScope = "prompt"
+)
+
+type Tag struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name" yaml:"name"`
+ Category string `bun:",notnull" json:"category" yaml:"category"`
+ Group string `bun:",notnull" json:"group" yaml:"group"`
+ Scope TagScope `bun:",notnull" json:"scope" yaml:"scope"`
+ BuiltIn bool `bun:",notnull" json:"built_in" yaml:"built_in"`
+ ShowName string `bun:"" json:"show_name" yaml:"show_name"`
+ times
+}
+
+// TagCategory represents the category of tags
+type TagCategory struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Name string `bun:",notnull" json:"name" yaml:"name"`
+ Scope TagScope `bun:",notnull" json:"scope" yaml:"scope"`
+}
+
+// Alltags returns all tags in the database
+func (ts *tagStoreImpl) AllTags(ctx context.Context) ([]Tag, error) {
+ var tags []Tag
+ err := ts.db.Operator.Core.NewSelect().Model(&Tag{}).Scan(ctx, &tags)
+ if err != nil {
+ slog.Error("Failed to select tags", "error", err)
+ return nil, err
+ }
+ return tags, nil
+}
+
+func (ts *tagStoreImpl) AllTagsByScope(ctx context.Context, scope TagScope) ([]*Tag, error) {
+ var tags []*Tag
+ err := ts.db.Operator.Core.NewSelect().Model(&tags).
+ Where("scope =?", scope).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to select tags by scope,cause: %w", err)
+ }
+ return tags, nil
+}
+
+func (ts *tagStoreImpl) AllTagsByScopeAndCategory(ctx context.Context, scope TagScope, category string) ([]*Tag, error) {
+ var tags []*Tag
+ query := ts.db.Operator.Core.NewSelect().Model(&tags)
+ if scope != "" {
+ query.Where("scope = ?", scope)
+ }
+ if category != "" {
+ query.Where("category = ?", category)
+ }
+
+ err := query.Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to select tags by scope,cause: %w", err)
+ }
+ return tags, nil
+}
+
+func (ts *tagStoreImpl) GetTagsByScopeAndCategories(ctx context.Context, scope TagScope, categories []string) ([]*Tag, error) {
+ var tags []*Tag
+ err := ts.db.Operator.Core.NewSelect().Model(&tags).
+ Where("scope = ? and category in (?)", scope, bun.In(categories)).
+ Scan(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to select tags by scope,cause: %w", err)
+ }
+ return tags, nil
+}
+
+func (ts *tagStoreImpl) AllModelTags(ctx context.Context) ([]*Tag, error) {
+ return ts.AllTagsByScope(ctx, ModelTagScope)
+}
+
+func (ts *tagStoreImpl) AllPromptTags(ctx context.Context) ([]*Tag, error) {
+ return ts.AllTagsByScope(ctx, PromptTagScope)
+}
+
+func (ts *tagStoreImpl) AllDatasetTags(ctx context.Context) ([]*Tag, error) {
+ return ts.AllTagsByScope(ctx, DatasetTagScope)
+}
+
+func (ts *tagStoreImpl) AllCodeTags(ctx context.Context) ([]*Tag, error) {
+ return ts.AllTagsByScope(ctx, CodeTagScope)
+}
+
+func (ts *tagStoreImpl) AllSpaceTags(ctx context.Context) ([]*Tag, error) {
+ return ts.AllTagsByScope(ctx, SpaceTagScope)
+}
+
+func (ts *tagStoreImpl) AllModelCategories(ctx context.Context) ([]TagCategory, error) {
+ return ts.allCategories(ctx, ModelTagScope)
+}
+
+func (ts *tagStoreImpl) AllPromptCategories(ctx context.Context) ([]TagCategory, error) {
+ return ts.allCategories(ctx, PromptTagScope)
+}
+
+func (ts *tagStoreImpl) AllDatasetCategories(ctx context.Context) ([]TagCategory, error) {
+ return ts.allCategories(ctx, DatasetTagScope)
+}
+
+func (ts *tagStoreImpl) AllCodeCategories(ctx context.Context) ([]TagCategory, error) {
+ return ts.allCategories(ctx, CodeTagScope)
+}
+
+func (ts *tagStoreImpl) AllSpaceCategories(ctx context.Context) ([]TagCategory, error) {
+ return ts.allCategories(ctx, SpaceTagScope)
+}
+
+func (ts *tagStoreImpl) allCategories(ctx context.Context, scope TagScope) ([]TagCategory, error) {
+ var tags []TagCategory
+ err := ts.db.Operator.Core.NewSelect().Model(&TagCategory{}).
+ Where("scope = ?", scope).
+ Scan(ctx, &tags)
+ if err != nil {
+ slog.Error("Failed to select tags", "error", err)
+ return nil, err
+ }
+ return tags, nil
+}
+
+func (ts *tagStoreImpl) CreateTag(ctx context.Context, category, name, group string, scope TagScope) (Tag, error) {
+ tag := Tag{
+ Name: name,
+ Category: category,
+ Group: group,
+ Scope: scope,
+ }
+ _, err := ts.db.Operator.Core.NewInsert().Model(&tag).Exec(ctx)
+ return tag, err
+}
+
+func (ts *tagStoreImpl) SaveTags(ctx context.Context, tags []*Tag) error {
+ if len(tags) == 0 {
+ return nil
+ }
+ _, err := ts.db.Operator.Core.NewInsert().Model(&tags).Exec(ctx)
+ if err != nil {
+ slog.Error("Failed to save tags", slog.Any("tags", tags), slog.Any("error", err))
+ return fmt.Errorf("failed to save tags,cause: %w", err)
+ }
+ return nil
+}
+
+// SetMetaTags will delete existing tags and create new ones
+func (ts *tagStoreImpl) SetMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string, tags []*Tag) (repoTags []*RepositoryTag, err error) {
+ repo := new(Repository)
+ err = ts.db.Operator.Core.NewSelect().Model(repo).
+ Column("id").
+ Relation("Tags").
+ Where("LOWER(git_path) = LOWER(?)", fmt.Sprintf("%ss_%v/%v", string(repoType), namespace, name)).
+ Scan(ctx)
+ if err != nil {
+ return repoTags, fmt.Errorf("failed to find repository, path:%v/%v,error:%w", namespace, name, err)
+ }
+
+ var metaTagIds []int64
+ exCategories := map[string]bool{
+ "framework": true,
+ "runtime_framework": true,
+ "evaluation": true,
+ }
+ for _, tag := range repo.Tags {
+ if !exCategories[tag.Category] {
+ metaTagIds = append(metaTagIds, tag.ID)
+ }
+ }
+ // select all repo tags which are not Library tags
+ err = ts.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ // remove all tags of the repository not belongs to category "Library", and then add new tags
+ _, err := tx.NewDelete().
+ Model(&RepositoryTag{}).
+ Where("repository_id =? and tag_id in (?)", repo.ID, bun.In(metaTagIds)).
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ // no new tags to insert
+ if len(tags) == 0 {
+ return nil
+ }
+ for _, tag := range tags {
+ if tag.Category == "framework" {
+ return errors.New("found framework tag when set meta tag, tag name:" + tag.Name)
+ }
+ repoTag := &RepositoryTag{RepositoryID: repo.ID, TagID: tag.ID, Repository: repo, Tag: tag, Count: 1}
+ repoTags = append(repoTags, repoTag)
+ }
+ // batch insert
+ _, err = tx.NewInsert().Model(&repoTags).Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to batch insert repository meta tags, path:%v/%v,error:%w", namespace, name, err)
+ }
+ return nil
+ })
+
+ return repoTags, err
+}
+
+func (ts *tagStoreImpl) SetLibraryTag(ctx context.Context, repoType types.RepositoryType, namespace, name string, newTag, oldTag *Tag) (err error) {
+ slog.Debug("set library tag", slog.Any("newTag", newTag), slog.Any("oldTag", oldTag))
+ repo := new(Repository)
+ err = ts.db.Operator.Core.NewSelect().Model(repo).
+ Column("id").
+ Where("LOWER(git_path) = LOWER(?)", fmt.Sprintf("%ss_%v/%v", string(repoType), namespace, name)).
+ Scan(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to find repository, path:%v/%v,error:%w", namespace, name, err)
+ }
+ err = ts.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ var err error
+ // decrease count of old tag
+ if oldTag != nil {
+ oldRepoTag := RepositoryTag{RepositoryID: repo.ID, TagID: oldTag.ID}
+ _, err = tx.NewUpdate().Model(&oldRepoTag).
+ Set("count = count-1").
+ Where("repository_id = ? and tag_id = ? and count > 0", repo.ID, oldTag.ID).
+ Exec(ctx)
+ if err != nil {
+ return err
+ }
+ }
+ // increase count of new tag
+ if newTag != nil {
+ newRepoTag := RepositoryTag{RepositoryID: repo.ID, TagID: newTag.ID}
+ err = tx.NewSelect().Model(&newRepoTag).
+ Where("repository_id = ? and tag_id = ?", repo.ID, newTag.ID).
+ Scan(ctx)
+
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return err
+ }
+ if newRepoTag.ID == 0 {
+ newRepoTag.Count = 1
+ _, err = tx.NewInsert().Model(&newRepoTag).Exec(ctx)
+ } else {
+ _, err = tx.NewUpdate().Model(&newRepoTag).Where("id = ?", newRepoTag.ID).Set("count = count+1").Exec(ctx)
+ }
+ }
+ return err
+ })
+ if err != nil {
+ slog.Error("Failed to update repository library tags", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("oldTag", oldTag), slog.Any("newTag", newTag), slog.Any("error", err))
+ return fmt.Errorf("failed to update repository library tags, path:%v/%v,error:%w", namespace, name, err)
+ }
+
+ return err
+}
+
+func (ts *tagStoreImpl) UpsertRepoTags(ctx context.Context, repoID int64, oldTagIDs, newTagIDs []int64) (err error) {
+ err = ts.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ var err error
+ if len(oldTagIDs) > 0 {
+ for _, tagID := range oldTagIDs {
+ _, err = tx.NewUpdate().Model((*RepositoryTag)(nil)).
+ Where("repository_id = ? and tag_id = ? and count > 0", repoID, tagID).
+ Set("count = count-1").
+ Exec(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to delete repository tags,error:%w", err)
+ }
+ }
+ }
+ // increase count of new tag
+ if len(newTagIDs) > 0 {
+ for _, tagID := range newTagIDs {
+ newRepoTag := RepositoryTag{
+ RepositoryID: repoID,
+ TagID: tagID,
+ Count: 1,
+ }
+ _, err = tx.NewInsert().Model(&newRepoTag).
+ On("CONFLICT (repository_id, tag_id) DO UPDATE SET count = repository_tag.count+1").
+ Exec(ctx)
+
+ if err != nil {
+ return fmt.Errorf("failed to upsert repository tags,error:%w", err)
+ }
+
+ }
+ }
+ return nil
+ })
+
+ return err
+}
+
+func (ts *tagStoreImpl) RemoveRepoTags(ctx context.Context, repoID int64, tagIDs []int64) (err error) {
+ if len(tagIDs) == 0 {
+ return nil
+ }
+ _, err = ts.db.Operator.Core.NewDelete().
+ Model(&RepositoryTag{}).
+ Where("repository_id =? and tag_id in (?)", repoID, bun.In(tagIDs)).
+ Exec(ctx)
+
+ return err
+}
+
+func (ts *tagStoreImpl) FindOrCreate(ctx context.Context, tag Tag) (*Tag, error) {
+ var resTag Tag
+ err := ts.db.Operator.Core.NewSelect().
+ Model(&resTag).
+ Where("name = ? and category = ? and built_in = ? and scope = ?", tag.Name, tag.Category, tag.BuiltIn, tag.Scope).
+ Scan(ctx)
+ if err == nil {
+ return &resTag, nil
+ }
+ _, err = ts.db.Operator.Core.NewInsert().
+ Model(&tag).
+ Exec(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &tag, err
+}
+
+// find tag by name
+func (ts *tagStoreImpl) FindTag(ctx context.Context, name, scope, category string) (*Tag, error) {
+ var tag Tag
+ err := ts.db.Operator.Core.NewSelect().
+ Model(&tag).
+ Where("name = ? and scope = ? and category = ?", name, scope, category).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &tag, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "time"
+)
+
+type tagRuleStoreImpl struct {
+ db *DB
+}
+
+type TagRuleStore interface {
+ // find dataset tag by name and related tag with tag name
+ FindByRepo(ctx context.Context, category, namespace, repoName, repoType string) (*TagRule, error)
+}
+
+func NewTagRuleStore() TagRuleStore {
+ return &tagRuleStoreImpl{db: defaultDB}
+}
+
+func NewTagRuleStoreWithDB(db *DB) TagRuleStore {
+ return &tagRuleStoreImpl{db: db}
+}
+
+// internal use for the relation between dataset and tag
+type TagRule struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ Namespace string `bun:",notnull" json:"namespace"`
+ RepoName string `bun:",notnull" json:"repo_name"`
+ RepoType string `bun:",notnull" json:"repo_type"`
+ Category string `bun:",notnull" json:"category"`
+ TagName string `bun:",notnull" json:"tag_name"`
+ RuntimeFramework string `bun:"," json:"runtime_framework"`
+ Source string `bun:"," json:"source"`
+ Tag Tag `bun:",rel:has-one,join:tag_name=name"`
+ CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"`
+}
+
+// find dataset tag by name and related tag with tag name
+func (s *tagRuleStoreImpl) FindByRepo(ctx context.Context, category, namespace, repoName, repoType string) (*TagRule, error) {
+ var TagRule TagRule
+ err := s.db.Core.NewSelect().
+ Model(&TagRule).
+ Relation("Tag").
+ Where("LOWER(tag_rule.repo_name) = LOWER(?) AND tag_rule.namespace =? AND tag_rule.repo_type = ? AND tag_rule.category = ?", repoName, namespace, repoType, category).
+ Limit(1).
+ Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &TagRule, nil
+}
+
+
+
package database
+
+import (
+ "context"
+ "time"
+)
+
+type Telemetry struct {
+ UUID string `bun:"" json:"uuid"`
+ RecordedAt time.Time `bun:"" json:"recorded_at"`
+ Hostname string `bun:"" json:"hostname,omitempty"`
+ Version string `bun:"" json:"version"`
+ InstallationType string `bun:"" json:"installation_type,omitempty"`
+ ActiveUserCount int `bun:"" json:"active_user_count"`
+ Edition string `bun:"" json:"edition,omitempty"`
+ LicenseMD5 string `bun:"" json:"license_md5,omitempty"`
+ LicenseID int `bun:"" json:"license_id,omitempty"`
+ HistoricalMaxUsers int `bun:"" json:"historical_max_users,omitempty"`
+ Licensee interface{} `bun:"type:jsonb" json:"licensee,omitempty"`
+ LicenseUserCount int `bun:"" json:"license_user_count,omitempty"`
+ LicenseBillableUsers int `bun:"" json:"license_billable_users,omitempty"`
+ LicenseStartsAt time.Time `bun:"" json:"license_starts_at,omitempty"`
+ LicenseExpiresAt time.Time `bun:"" json:"license_expires_at,omitempty"`
+ LicensePlan string `bun:"" json:"license_plan,omitempty"`
+ LicenseAddOns map[string]interface{} `bun:"type:jsonb" json:"license_add_ons,omitempty"`
+ Settings interface{} `bun:"type:jsonb" json:"settings,omitempty"`
+ Counts interface{} `bun:"type:jsonb" json:"counts,omitempty"`
+}
+
+type telemetryStoreImpl struct {
+ db *DB
+}
+
+type TelemetryStore interface {
+ Save(ctx context.Context, telemetry *Telemetry) error
+}
+
+func NewTelemetryStore() TelemetryStore {
+ return &telemetryStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewTelemetryStoreWithDB(db *DB) TelemetryStore {
+ return &telemetryStoreImpl{
+ db: db,
+ }
+}
+
+func (s *telemetryStoreImpl) Save(ctx context.Context, telemetry *Telemetry) error {
+ return assertAffectedOneRow(s.db.Core.NewInsert().Model(telemetry).Exec(ctx))
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/uptrace/bun"
+)
+
+// Define the UserStore interface
+type UserStore interface {
+ Index(ctx context.Context) ([]User, error)
+ IndexWithSearch(ctx context.Context, search string, per, page int) ([]User, int, error)
+ FindByUsername(ctx context.Context, username string) (User, error)
+ Update(ctx context.Context, user *User) error
+ ChangeUserName(ctx context.Context, username string, newUsername string) error
+ Create(ctx context.Context, user *User, namespace *Namespace) error
+ IsExist(ctx context.Context, username string) (bool, error)
+ IsExistByUUID(ctx context.Context, uuid string) (bool, error)
+ FindByAccessToken(ctx context.Context, token string) (*User, error)
+ FindByGitAccessToken(ctx context.Context, token string) (*User, error)
+ FindByUUID(ctx context.Context, uuid string) (*User, error)
+ DeleteUserAndRelations(ctx context.Context, input User) error
+ CountUsers(ctx context.Context) (int, error)
+}
+
+// Implement the UserStore interface in UserStoreImpl
+type UserStoreImpl struct {
+ db *DB
+}
+
+func NewUserStore() UserStore {
+ return &UserStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewUserStoreWithDB(db *DB) UserStore {
+ return &UserStoreImpl{
+ db: db,
+ }
+}
+
+type User struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ GitID int64 `bun:",notnull" json:"git_id"`
+ NickName string `bun:"column:name,notnull" json:"name"`
+ Username string `bun:",notnull,unique" json:"username"`
+ Email string `bun:",nullzero,unique" json:"email"`
+ //git password
+ Password string `bun:",notnull" json:"-"`
+ AccessTokens []AccessToken `bun:"rel:has-many,join:id=user_id"`
+ Namespaces []Namespace `bun:"rel:has-many,join:id=user_id" json:"namespace"`
+ // TODO:add unique index after migration
+ UUID string `bun:"," json:"uuid"`
+ // user registered from default login page, from casdoor, etc. Possible values:
+ //
+ // - "default"
+ // - "casdoor"
+ RegProvider string `bun:"," json:"reg_provider"`
+ Gender string `bun:"," json:"gender"`
+ RoleMask string `bun:"," json:"role_mask"`
+ Phone string `bun:"," json:"phone"`
+ PhoneVerified bool `bun:"," json:"phone_verified"`
+ EmailVerified bool `bun:"," json:"email_verified"`
+ LastLoginAt string `bun:"," json:"last_login_at"`
+ Avatar string `bun:"," json:"avatar"`
+ CompanyVerified bool `bun:"," json:"company_verified"`
+ //password for user registered without casdoor
+ PasswordHash string `bun:"," json:"password_hash"`
+ Homepage string `bun:"," json:"homepage"`
+ Bio string `bun:"," json:"bio"`
+ // allow user to change username once
+ CanChangeUserName bool `bun:"can_change_user_name" json:"can_change_username"`
+
+ // WechatID string `bun:"," json:"wechat_id"`
+ // GithubID string `bun:"," json:"github_id"`
+ // GitlabID string `bun:"," json:"gitlab_id"`
+ // SessionIP string `bun:"," json:"session_ip"`
+ // Nickname string `bun:"," json:"nickname"`
+ // GitToken string `bun:"," json:"git_token"`
+ // StarhubSynced bool `bun:"," json:"starhub_synced"`
+ // GitTokenName string `bun:"," json:"git_token_name"`
+
+ times
+}
+
+func (u *User) Roles() []string {
+ if len(u.RoleMask) == 0 {
+ return []string{}
+ }
+ return strings.Split(u.RoleMask, ",")
+}
+
+// CanAdmin checks if the user has admin or super_user roles.
+// It returns a boolean indicating whether the user has admin or super_user roles.
+func (u *User) CanAdmin() bool {
+ return strings.Contains(u.RoleMask, "admin") || strings.Contains(u.RoleMask, "super_user")
+}
+
+func (u *User) SetRoles(roles []string) {
+ u.RoleMask = strings.Join(roles, ",")
+}
+
+func (s *UserStoreImpl) Index(ctx context.Context) (users []User, err error) {
+ err = s.db.Operator.Core.NewSelect().Model(&users).Scan(ctx, &users)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *UserStoreImpl) IndexWithSearch(ctx context.Context, search string, per, page int) (users []User, count int, err error) {
+ search = strings.ToLower(search)
+ query := s.db.Operator.Core.NewSelect().
+ Model(&users)
+ if search != "" {
+ query.Where("LOWER(username) like ? OR LOWER(email) like ?", fmt.Sprintf("%%%s%%", search), fmt.Sprintf("%%%s%%", search))
+ }
+ count, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ query.Order("id asc").Limit(per).Offset((page - 1) * per)
+ err = query.Scan(ctx, &users)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func (s *UserStoreImpl) FindByUsername(ctx context.Context, username string) (user User, err error) {
+ user.Username = username
+ err = s.db.Operator.Core.NewSelect().Model(&user).Where("username = ?", username).Scan(ctx)
+ return
+}
+
+func (s *UserStoreImpl) FindByID(ctx context.Context, id int) (user User, err error) {
+ user.ID = int64(id)
+ err = s.db.Operator.Core.NewSelect().Model(&user).WherePK().Scan(ctx)
+ return
+}
+
+func (s *UserStoreImpl) Update(ctx context.Context, user *User) (err error) {
+ err = assertAffectedOneRow(s.db.Operator.Core.NewUpdate().
+ Model(user).
+ WherePK().
+ Exec(ctx),
+ )
+
+ return
+}
+
+func (s *UserStoreImpl) ChangeUserName(ctx context.Context, username string, newUsername string) (err error) {
+ return s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err = assertAffectedOneRow(tx.NewUpdate().Model((*Namespace)(nil)).
+ Set("path = ?", newUsername).
+ Set("updated_at = now()").
+ Where("path = ?", username).
+ Exec(ctx)); err != nil {
+ return fmt.Errorf("failed to change namespace from '%s' to '%s' in db, error:%w", username, newUsername, err)
+ }
+
+ err = assertAffectedOneRow(tx.NewUpdate().
+ Model((*User)(nil)).
+ Where("username = ?", username).
+ Set("username = ?", newUsername).
+ Set("can_change_user_name = false").
+ Set("updated_at = now()").
+ Exec(ctx))
+ if err != nil {
+ return fmt.Errorf("failed to change username from '%s' to '%s' in db, error:%w", username, newUsername, err)
+ }
+ return nil
+ })
+}
+
+func (s *UserStoreImpl) Create(ctx context.Context, user *User, namespace *Namespace) (err error) {
+ err = s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ if err = assertAffectedOneRow(tx.NewInsert().Model(user).Exec(ctx)); err != nil {
+ return err
+ }
+ namespace.UserID = user.ID
+ namespace.NamespaceType = UserNamespace
+ if err = assertAffectedOneRow(tx.NewInsert().Model(namespace).Exec(ctx)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return
+}
+
+func (s *UserStoreImpl) IsExist(ctx context.Context, username string) (exists bool, err error) {
+ return s.db.Operator.Core.
+ NewSelect().
+ Model((*User)(nil)).
+ Where("username =?", username).
+ Exists(ctx)
+}
+
+func (s *UserStoreImpl) IsExistByUUID(ctx context.Context, uuid string) (exists bool, err error) {
+ return s.db.Operator.Core.
+ NewSelect().
+ Model((*User)(nil)).
+ Where("uuid =?", uuid).
+ Exists(ctx)
+}
+
+// FindByAccessToken retrieves user information based on the access token. The access token must be active and not expired.
+func (s *UserStoreImpl) FindByAccessToken(ctx context.Context, token string) (*User, error) {
+ var user User
+ _, err := s.db.Operator.Core.
+ NewSelect().
+ ColumnExpr("u.*").
+ TableExpr("users AS u").
+ Join("JOIN access_tokens AS t ON u.id = t.user_id").
+ Where("t.token = ? and t.is_active = true and (t.expired_at is null or t.expired_at > now()) ", token).
+ Exec(ctx, &user)
+ if err != nil {
+ return nil, err
+ }
+ return &user, nil
+}
+
+func (s *UserStoreImpl) FindByGitAccessToken(ctx context.Context, token string) (*User, error) {
+ var user User
+ _, err := s.db.Operator.Core.
+ NewSelect().
+ ColumnExpr("u.*").
+ TableExpr("users AS u").
+ Join("JOIN access_tokens AS t ON u.id = t.user_id").
+ Where("t.token = ? and t.is_active = true and (t.expired_at is null or t.expired_at > now()) and app = 'git'", token).
+ Exec(ctx, &user)
+
+ if err != nil {
+ return nil, err
+ }
+ return &user, nil
+}
+
+func (s *UserStoreImpl) FindByUUID(ctx context.Context, uuid string) (*User, error) {
+ var user User
+ err := s.db.Operator.Core.NewSelect().Model(&user).Where("uuid = ?", uuid).Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &user, nil
+}
+
+func (s *UserStoreImpl) GetActiveUserCount(ctx context.Context) (int, error) {
+ return s.db.Operator.Core.
+ NewSelect().
+ Model(&User{}).
+ Count(ctx)
+}
+
+func (s *UserStoreImpl) DeleteUserAndRelations(ctx context.Context, input User) (err error) {
+ exists, err := s.IsExist(ctx, input.Username)
+ if err != nil {
+ return fmt.Errorf("error checking if user exists: %v", err)
+ }
+ if !exists {
+ return fmt.Errorf("user does not exist")
+ }
+
+ err = s.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ // Delete user
+ if err = assertAffectedOneRow(tx.NewDelete().Model(&input).Where("id = ?", input.ID).Exec(ctx)); err != nil {
+ return fmt.Errorf("failed to delete user %d: %v", input.ID, err)
+ }
+ // Get user's repository_ids
+ var repoIDs []int64
+ if err := s.db.Operator.Core.NewSelect().Column("id").Model(&Repository{}).Where("user_id = ?", input.ID).Scan(ctx, &repoIDs); err != nil {
+ return fmt.Errorf("failed to get user repo ids: %v", err)
+ }
+ // Delete user's model
+ if _, err := tx.NewDelete().Model(&Model{}).Where("repository_id IN (?)", bun.In(repoIDs)).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user models for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's dataset
+ if _, err := tx.NewDelete().Model(&Dataset{}).Where("repository_id IN (?)", bun.In(repoIDs)).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user datasets for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's code
+ if _, err := tx.NewDelete().Model(&Code{}).Where("repository_id IN (?)", bun.In(repoIDs)).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user codes for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's space
+ if _, err := tx.NewDelete().Model(&Space{}).Where("repository_id IN (?)", bun.In(repoIDs)).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user spaces for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's namespace
+ if _, err := tx.NewDelete().Model(&Namespace{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user namespace for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's prompts
+ if _, err := tx.NewDelete().Model(&Prompt{}).Where("repository_id IN (?)", bun.In(repoIDs)).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user prompts for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's repositories runtime frameworks
+ if _, err := tx.NewDelete().Model(&RepositoriesRuntimeFramework{}).Where("repo_id IN (?)", bun.In(repoIDs)).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user repositories runtime frameworks for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's repo
+ if _, err = tx.NewDelete().Model(&Repository{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user repos for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's access token
+ if _, err := tx.NewDelete().Model(&AccessToken{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user access tokens for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's organization
+ if _, err := tx.NewDelete().Model(&Organization{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user organizations for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's member
+ if _, err := tx.NewDelete().Model(&Member{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user members for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's account_sync_quota
+ if _, err := tx.NewDelete().Model(&AccountSyncQuota{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user account sync quotas for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's ssh keys
+ if _, err := tx.NewDelete().Model(&SSHKey{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user ssh keys for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's user likes
+ if _, err := tx.NewDelete().Model(&UserLike{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user likes for user ID %d: %v", input.ID, err)
+ }
+ // Delete user's prompt conversations
+ if _, err := tx.NewDelete().Model(&PromptConversation{}).Where("user_id = ?", input.ID).Exec(ctx); err != nil {
+ return fmt.Errorf("failed to delete user prompt conversations for user ID %d: %v", input.ID, err)
+ }
+
+ return nil
+ })
+ return
+}
+
+func (s *UserStoreImpl) CountUsers(ctx context.Context) (int, error) {
+ var users []User
+ q := s.db.Operator.Core.NewSelect().Model(&users)
+ count, err := q.Count(ctx)
+ if err != nil {
+ return 0, fmt.Errorf("failed to count users: %w", err)
+ }
+ return count, nil
+}
+
+
+
package database
+
+import (
+ "context"
+
+ "github.com/uptrace/bun"
+)
+
+type userLikesStoreImpl struct {
+ db *DB
+}
+
+type UserLikesStore interface {
+ Add(ctx context.Context, userId, repoId int64) error
+ LikeCollection(ctx context.Context, userId, collectionId int64) error
+ UnLikeCollection(ctx context.Context, userId, collectionId int64) error
+ Delete(ctx context.Context, userId, repoId int64) error
+ IsExist(ctx context.Context, username string, repoId int64) (exists bool, err error)
+ IsExistCollection(ctx context.Context, username string, collectionId int64) (exists bool, err error)
+}
+
+func NewUserLikesStore() UserLikesStore {
+ return &userLikesStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewUserLikesStoreWithDB(db *DB) UserLikesStore {
+ return &userLikesStoreImpl{
+ db: db,
+ }
+}
+
+type UserLike struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserID int64 `bun:",notnull" json:"user_id"`
+ RepoID int64 `bun:",notnull" json:"repo_id"`
+ CollectionID int64 `bun:",notnull" json:"collection_id"`
+}
+
+func (r *userLikesStoreImpl) Add(ctx context.Context, userId, repoId int64) error {
+ err := r.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ userLikes := &UserLike{
+ UserID: userId,
+ RepoID: repoId,
+ }
+ if err := assertAffectedOneRow(tx.NewInsert().Model(userLikes).Exec(ctx)); err != nil {
+ return err
+ }
+
+ if err := assertAffectedOneRow(tx.Exec("update repositories set likes=COALESCE(likes, 0)+1 where id=?", repoId)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return err
+}
+
+func (r *userLikesStoreImpl) LikeCollection(ctx context.Context, userId, collectionId int64) error {
+ err := r.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ userLikes := &UserLike{
+ UserID: userId,
+ CollectionID: collectionId,
+ }
+ if err := assertAffectedOneRow(tx.NewInsert().Model(userLikes).Exec(ctx)); err != nil {
+ return err
+ }
+
+ if err := assertAffectedOneRow(tx.Exec("update collections set likes=COALESCE(likes, 0)+1 where id=?", collectionId)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return err
+}
+
+func (r *userLikesStoreImpl) UnLikeCollection(ctx context.Context, userId, collectionId int64) error {
+ err := r.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ var userLikes UserLike
+ if err := assertAffectedOneRow(r.db.Core.NewDelete().Model(&userLikes).Where("user_id = ? and collection_id = ?", userId, collectionId).Exec(ctx)); err != nil {
+ return err
+ }
+
+ if err := assertAffectedOneRow(tx.Exec("update collections set likes=COALESCE(likes, 1)-1 where id=?", collectionId)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return err
+}
+
+func (r *userLikesStoreImpl) Delete(ctx context.Context, userId, repoId int64) error {
+ err := r.db.Operator.Core.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
+ var userLikes UserLike
+ if err := assertAffectedOneRow(r.db.Core.NewDelete().Model(&userLikes).Where("user_id = ? and repo_id = ?", userId, repoId).Exec(ctx)); err != nil {
+ return err
+ }
+
+ if err := assertAffectedOneRow(tx.Exec("update repositories set likes=COALESCE(likes, 1)-1 where id=?", repoId)); err != nil {
+ return err
+ }
+ return nil
+ })
+ return err
+}
+
+func (r *userLikesStoreImpl) IsExist(ctx context.Context, username string, repoId int64) (exists bool, err error) {
+ var userLike UserLike
+ exists, err = r.db.Operator.Core.
+ NewSelect().
+ Model(&userLike).
+ Join("JOIN users ON users.id = user_like.user_id").
+ Where("user_like.repo_id = ? and users.username = ?", repoId, username).
+ Exists(ctx)
+ return
+}
+
+func (r *userLikesStoreImpl) IsExistCollection(ctx context.Context, username string, collectionId int64) (exists bool, err error) {
+ var userLike UserLike
+ exists, err = r.db.Operator.Core.
+ NewSelect().
+ Model(&userLike).
+ Join("JOIN users ON users.id = user_like.user_id").
+ Where("user_like.collection_id = ? and users.username = ?", collectionId, username).
+ Exists(ctx)
+ return
+}
+
+
+
package database
+
+import (
+ "context"
+ "fmt"
+ "time"
+)
+
+type userResourcesStoreImpl struct {
+ db *DB
+}
+
+type UserResourcesStore interface {
+ // add user resources
+ AddUserResources(ctx context.Context, userResources *UserResources) error
+ // get user resources by user uid
+ GetUserResourcesByUserUID(ctx context.Context, per, page int, userId string) (userResources []UserResources, total int, err error)
+ // get need reserved user resources which is not deployed and not expired
+ GetReservedUserResources(ctx context.Context, userId string, clusterId string) ([]UserResources, error)
+ // update deploy id
+ UpdateDeployId(ctx context.Context, userResources *UserResources) error
+ // find user resources by order detail id
+ FindUserResourcesByOrderDetailId(ctx context.Context, userUId string, orderDetailId int64) (*UserResources, error)
+ // delete user resources by order detail id
+ DeleteUserResourcesByOrderDetailId(ctx context.Context, userUid string, orderDetailId int64) error
+}
+
+func NewUserResourcesStore() UserResourcesStore {
+ return &userResourcesStoreImpl{
+ db: defaultDB,
+ }
+}
+
+func NewUserResourcesStoreWithDB(db *DB) UserResourcesStore {
+ return &userResourcesStoreImpl{
+ db: db,
+ }
+}
+
+type UserResources struct {
+ ID int64 `bun:",pk,autoincrement" json:"id"`
+ UserUID string `bun:",notnull" json:"user_uid"`
+ OrderId string `bun:",notnull" json:"order_id"`
+ OrderDetailId int64 `bun:",notnull,unique" json:"order_detail_id"`
+ ResourceId int64 `bun:",notnull" json:"resource_id"`
+ DeployId int64 `bun:",notnull" json:"deploy_id"`
+ XPUNum int `bun:",notnull" json:"xpu_num"`
+ PayMode string `bun:",notnull" json:"pay_mode"`
+ Price float64 `bun:",notnull" json:"price"`
+ CreatedAt time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"created_at"`
+ StartTime time.Time `bun:",nullzero,notnull,skipupdate,default:current_timestamp" json:"start_time"`
+ EndTime time.Time `bun:",nullzero,notnull,skipupdate" json:"end_time"`
+ SpaceResource *SpaceResource `bun:"rel:belongs-to,join:resource_id=id" json:"resource"`
+ Deploy *Deploy `bun:"rel:belongs-to,join:deploy_id=id" json:"deploy"`
+}
+
+// add user resources
+func (s *userResourcesStoreImpl) AddUserResources(ctx context.Context, userResources *UserResources) error {
+ res, err := s.db.Core.NewInsert().Model(userResources).Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("failed to create user resource in db ,error:%w", err)
+ }
+ return nil
+}
+
+// get user resources by user uid
+func (s *userResourcesStoreImpl) GetUserResourcesByUserUID(ctx context.Context, per, page int, userId string) (userResources []UserResources, total int, err error) {
+
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&userResources).
+ Relation("SpaceResource").
+ Relation("Deploy").
+ Where("user_resources.user_uid = ?", userId)
+
+ query = query.Order("user_resources.created_at DESC").
+ Limit(per).
+ Offset((page - 1) * per)
+
+ err = query.Scan(ctx)
+ if err != nil {
+ return
+ }
+ total, err = query.Count(ctx)
+ if err != nil {
+ return
+ }
+ return
+}
+
+// get need reserved user resources which is not deployed and not expired
+func (s *userResourcesStoreImpl) GetReservedUserResources(ctx context.Context, userId string, clusterId string) ([]UserResources, error) {
+ var userResources []UserResources
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&userResources).
+ Relation("SpaceResource").
+ Where("user_resources.deploy_id = ?", 0).
+ Where("user_resources.end_time > ?", time.Now())
+ if userId != "" {
+ query.Where("user_resources.user_uid = ?", userId)
+ }
+ err := query.Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if clusterId == "" {
+ return userResources, nil
+ }
+ var filteredUserResources []UserResources
+ for _, userResource := range userResources {
+ if userResource.SpaceResource.ClusterID == clusterId {
+ filteredUserResources = append(filteredUserResources, userResource)
+ }
+ }
+ return filteredUserResources, nil
+}
+
+// update deploy id
+func (s *userResourcesStoreImpl) UpdateDeployId(ctx context.Context, userResources *UserResources) error {
+ res, err := s.db.Core.NewUpdate().Model(userResources).WherePK().Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("failed to update user resource in db ,error:%w", err)
+ }
+ return nil
+}
+
+// find user resources by order detail id
+func (s *userResourcesStoreImpl) FindUserResourcesByOrderDetailId(ctx context.Context, userUId string, orderDetailId int64) (*UserResources, error) {
+ var userResources UserResources
+ query := s.db.Operator.Core.
+ NewSelect().
+ Model(&userResources).
+ Where("user_resources.order_detail_id = ?", orderDetailId)
+ if userUId != "" {
+ query.Where("user_resources.user_uid = ?", userUId)
+ }
+
+ err := query.Scan(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &userResources, nil
+}
+
+// delete user resources by order detail id
+func (s *userResourcesStoreImpl) DeleteUserResourcesByOrderDetailId(ctx context.Context, userUid string, orderDetailId int64) error {
+ res, err := s.db.Core.NewDelete().Model(&UserResources{}).
+ Where("order_detail_id = ?", orderDetailId).
+ Where("user_uid = ?", userUid).
+ Exec(ctx)
+ if err := assertAffectedOneRow(res, err); err != nil {
+ return fmt.Errorf("failed to delete user resource in db ,error:%w", err)
+ }
+ return nil
+}
+
+
+
package s3
+
+import (
+ "context"
+ "io"
+ "log/slog"
+ "net/url"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/minio/minio-go/v7/pkg/credentials"
+ "opencsg.com/csghub-server/builder/geo"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func NewMinio(cfg *config.Config) (Client, error) {
+ mClient, err := minio.New(cfg.S3.Endpoint, &minio.Options{
+ Creds: credentials.NewStaticV4(cfg.S3.AccessKeyID, cfg.S3.AccessKeySecret, ""),
+ Secure: cfg.S3.EnableSSL,
+ BucketLookup: minio.BucketLookupAuto,
+ Region: cfg.S3.Region,
+ })
+ if err != nil {
+ return nil, err
+ }
+ geo.Config(cfg)
+ return &minioClient{Client: *mClient}, nil
+}
+
+type Client interface {
+ PresignedGetObjectGeo(ctx context.Context, bucketName, objectName string, expires time.Duration, reqParams url.Values) (*url.URL, error)
+ PutObject(ctx context.Context, bucketName, objectName string, reader io.Reader, objectSize int64,
+ opts minio.PutObjectOptions,
+ ) (info minio.UploadInfo, err error)
+ StatObject(ctx context.Context, bucketName, objectName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error)
+ RemoveObject(ctx context.Context, bucketName, objectName string, opts minio.RemoveObjectOptions) error
+ PresignedPutObject(ctx context.Context, bucketName, objectName string, expires time.Duration) (u *url.URL, err error)
+}
+
+type minioClient struct {
+ minio.Client
+}
+
+// PresignedGetObjectGeo is a wrapper around minio.Client.PresignedGetObject that adds geo CDN support
+//
+// It gets the client IP from the context and uses the geo package to get the ip location and the nearest CDN domain
+func (c *minioClient) PresignedGetObjectGeo(ctx context.Context, bucketName, objectName string, expires time.Duration, reqParams url.Values) (*url.URL, error) {
+ originalUrl, err := c.PresignedGetObject(ctx, bucketName, objectName, expires, reqParams)
+ if err != nil {
+ return nil, err
+ }
+ //get client ip from ctx
+ clientIP, ok := ctx.Value("clientIP").(string)
+ if !ok {
+ return originalUrl, nil
+ }
+
+ cdnUrl, err := geo.CDNUrl(clientIP, originalUrl)
+ if err != nil {
+ // if fail to get the CDN url, just return the original url
+ slog.Error("fail to get cdn url for oss object", slog.Any("err", err), slog.String("client_id", clientIP))
+ return originalUrl, nil
+ }
+ return cdnUrl, nil
+}
+
+
+
package accounting
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(launchCmd)
+}
+
+var Cmd = &cobra.Command{
+ Use: "accounting",
+ Short: "entry point for accounting",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package accounting
+
+import (
+ "fmt"
+ "github.com/spf13/cobra"
+ "log/slog"
+ "opencsg.com/csghub-server/accounting/consumer"
+ "opencsg.com/csghub-server/accounting/router"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/mq"
+)
+
+var launchCmd = &cobra.Command{
+ Use: "launch",
+ Short: "Launch accounting server",
+ Example: serverExample(),
+ RunE: func(cmd *cobra.Command, args []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ slog.Debug("config", slog.Any("data", cfg))
+ // Check APIToken length
+ if len(cfg.APIToken) < 128 {
+ return fmt.Errorf("API token length is less than 128, please check")
+ }
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+
+ mqHandler, err := mq.Init(cfg)
+ if err != nil {
+ return fmt.Errorf("fail to build message queue handler: %w", err)
+ }
+
+ // Do metering
+ meter := consumer.NewMetering(mqHandler, cfg)
+ meter.Run()
+
+ rechargeConsumer := consumer.NewRechargeConsumer(mqHandler, cfg)
+ rechargeConsumer.Run()
+
+ if cfg.Accounting.ChargingEnable {
+ // Do charging
+ slog.Debug("enable charging")
+ charge := consumer.NewCharging(mqHandler, cfg)
+ charge.Run()
+ }
+
+ r, err := router.NewAccountRouter(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+ slog.Info("http server is running", slog.Any("port", cfg.Accounting.Port))
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.Accounting.Port,
+ },
+ r,
+ )
+ server.Run()
+
+ return nil
+ },
+}
+
+func serverExample() string {
+ return `
+# for development
+csghub-server accounting launch
+`
+}
+
+
+
package cron
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+var cmdCalcRecomScore = &cobra.Command{
+ Use: "calc-recom-score",
+ Short: "the cmd to calculate repository recommendation score",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+ c, err := component.NewRecomComponent(config)
+ if err != nil {
+ slog.Error("failed to create recom component", "err", err)
+ return
+ }
+ c.CalculateRecomScore(cmd.Context())
+ },
+}
+
+
+
package cron
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+var cmdCreatePushMirror = &cobra.Command{
+ Use: "create-push-mirror",
+ Short: "the cmd to create push mirror for mirrored repositories",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+ if !config.Saas {
+ return
+ }
+
+ if config.GitServer.Type != types.GitServerTypeGitea {
+ return
+ }
+ c, err := component.NewMirrorComponent(config)
+ if err != nil {
+ slog.Error("failed to create mirror component", "err", err)
+ return
+ }
+ err = c.CreatePushMirrorForFinishedMirrorTask(cmd.Context())
+ if err != nil {
+ slog.Error("failed to create push mirror task", "err", err)
+ return
+ }
+ },
+}
+
+
+
package cron
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(cmdCalcRecomScore)
+ Cmd.AddCommand(cmdCreatePushMirror)
+ Cmd.AddCommand(cmdGenTelemetry)
+}
+
+var Cmd = &cobra.Command{
+ Use: "cron",
+ Short: "entry point for cron jobs",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package cron
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "time"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/cache"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+var cmdGenTelemetry = &cobra.Command{
+ Use: "gen-telemetry",
+ Short: "the cmd to generate telemetry data",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+ //Saas don't need to generate telemetry data
+ if config.Saas {
+ return
+ }
+
+ locker, err := cache.NewCache(ctx, cache.RedisConfig{
+ Addr: config.Redis.Endpoint,
+ Username: config.Redis.User,
+ Password: config.Redis.Password,
+ })
+
+ if err != nil {
+ slog.Error("failed to initialize redis", "err", err)
+ return
+ }
+
+ if err = locker.RunWhileLocked(ctx, "gen-telemetry-lock", 10*time.Minute, genTelemetry(config)); err != nil {
+ slog.Error("failed to run gen telemetry", "err", err)
+ return
+ }
+ },
+}
+
+func genTelemetry(config *config.Config) func(ctx context.Context) error {
+ return func(ctx context.Context) error {
+ c, err := component.NewTelemetryComponent()
+ if err != nil {
+ return fmt.Errorf("failed to create mirror component, %w", err)
+ }
+ usage, err := c.GenUsageData(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to generate usage data, %w", err)
+ }
+
+ // save to local storage first
+ err = c.SaveUsageData(ctx, usage)
+ if err != nil {
+ return fmt.Errorf("failed to save usage data to local storage, %w", err)
+ }
+
+ if !config.Telemetry.Enable {
+ slog.Info("telemetry is not allowed to report")
+ return nil
+ }
+
+ // send report to telemetry server
+ hc := http.DefaultClient
+ var data bytes.Buffer
+ err = json.NewEncoder(&data).Encode(usage)
+ if err != nil {
+ return fmt.Errorf("failed to encode usage data, %w", err)
+ }
+
+ teleUrl, err := url.JoinPath(config.Telemetry.ReportURL, "usage")
+ if err != nil {
+ return fmt.Errorf("failed to join telemetry url, %w", err)
+ }
+ resp, err := hc.Post(teleUrl, "application/json", bytes.NewReader(data.Bytes()))
+ if err != nil {
+ return fmt.Errorf("failed to report usage data, %w", err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ var respData bytes.Buffer
+ _, err = io.Copy(&respData, resp.Body)
+ if err != nil {
+ return fmt.Errorf("io copy failed %w", err)
+ }
+ return fmt.Errorf("telemetry api returns error,url:%s, status:%d, body:%s", teleUrl, resp.StatusCode, respData.String())
+ }
+ return nil
+ }
+}
+
+
+
package deploy
+
+import (
+ "github.com/spf13/cobra"
+)
+
+var startBuilderCmd = &cobra.Command{
+ Use: "builder",
+ Short: "start space builder service",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ // config, err := config.LoadConfig()
+ // if err != nil {
+ // return
+ // }
+
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package deploy
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/runner/router"
+)
+
+var startRunnerCmd = &cobra.Command{
+ Use: "runner",
+ Short: "start runner service",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ return
+ },
+ RunE: func(cmd *cobra.Command, args []string) error {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+ err = event.InitEventPublisher(config, nil)
+ if err != nil {
+ return fmt.Errorf("fail to initialize message queue, %w", err)
+ }
+ s, err := router.NewHttpServer(config)
+ if err != nil {
+ return err
+ }
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: config.Space.RunnerServerPort,
+ },
+ s,
+ )
+ server.Run()
+ return nil
+ },
+}
+
+
+
package deploy
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ Cmd.AddCommand(
+ startBuilderCmd,
+ startRunnerCmd,
+ )
+}
+
+var Cmd = &cobra.Command{
+ Use: "deploy",
+ Short: "entry point of space builder and runner",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ // config, err := config.LoadConfig()
+ // if err != nil {
+ // return
+ // }
+
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package git
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/s3"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var generateLfsMetaObjectsCmd = &cobra.Command{
+ Use: "generate-lfs-meta-objects",
+ Short: "the cmd to generate lfs meta objects",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+
+ if config.GitServer.Type == types.GitServerTypeGitea {
+ return
+ }
+
+ s3Client, err := s3.NewMinio(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to init s3 client for code,error:%w", err)
+ slog.Error(newError.Error())
+ return
+ }
+
+ lfsMetaObjectStore := database.NewLfsMetaObjectStore()
+ repoStore := database.NewRepoStore()
+
+ gitServer, err := git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return
+ }
+
+ var i int
+ for {
+ queryCtx, queryCancel := context.WithTimeout(context.Background(), time.Second*15)
+ defer queryCancel()
+ repos, err := repoStore.FindWithBatch(queryCtx, 1000, i)
+ i += 1
+ if err != nil {
+ slog.Error("fail to batch get repositories", slog.Any("err", err))
+ return
+ }
+ if len(repos) == 0 {
+ break
+ }
+ for _, repo := range repos {
+ err := fetchAllPointersForRepo(config, gitServer, s3Client, lfsMetaObjectStore, repo)
+ if err != nil {
+ slog.Error("fail to fetch all pointers for repository", slog.Any("err", err))
+ continue
+ }
+ }
+ }
+ },
+}
+
+func fetchAllPointersForRepo(config *config.Config, gitServer gitserver.GitServer, s3Client s3.Client, lfsMetaObjectStore database.LfsMetaObjectStore, repo database.Repository) error {
+ namespace := strings.Split(repo.Path, "/")[0]
+ name := strings.Split(repo.Path, "/")[1]
+ ref := repo.DefaultBranch
+ if ref == "" {
+ ref = "main"
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
+ defer cancel()
+ slog.Info("start to fetch all pointers for repository", slog.String("namespace", namespace), slog.String("name", name), slog.String("ref", ref))
+ lfsPointers, err := gitServer.GetRepoAllLfsPointers(ctx, gitserver.GetRepoAllFilesReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: ref,
+ RepoType: repo.RepositoryType,
+ })
+ if err != nil {
+ return err
+ }
+
+ for _, lfsPointer := range lfsPointers {
+ pointer := types.Pointer{
+ Oid: lfsPointer.FileOid,
+ Size: lfsPointer.FileSize,
+ }
+ checkAndUpdateLfsMetaObjects(config, s3Client, lfsMetaObjectStore, repo, &pointer)
+ }
+ return nil
+}
+
+func checkAndUpdateLfsMetaObjects(config *config.Config, s3Client s3.Client, lfsMetaObjectStore database.LfsMetaObjectStore, repo database.Repository, pointer *types.Pointer) {
+ var exists bool
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ objectKey := path.Join("lfs", pointer.RelativePath())
+ _, err := s3Client.StatObject(ctx, config.S3.Bucket, objectKey, minio.StatObjectOptions{})
+ if err != nil {
+ slog.Error("failed to check if lfs file exists", slog.String("oid", objectKey), slog.Any("error", err))
+ exists = false
+ } else {
+ exists = true
+ }
+ slog.Info("lfs file exists", slog.Bool("exists", exists))
+ _, err = lfsMetaObjectStore.UpdateOrCreate(ctx, database.LfsMetaObject{
+ Oid: pointer.Oid,
+ Size: pointer.Size,
+ RepositoryID: repo.ID,
+ Existing: exists,
+ })
+ if err != nil {
+ slog.Error("lfsMetaObjectStore UpdateOrCreate failed", "error", err)
+ }
+}
+
+
+
package git
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ Cmd.AddCommand(generateLfsMetaObjectsCmd)
+}
+
+var Cmd = &cobra.Command{
+ Use: "git",
+ Short: "git related commands",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package logscan
+
+import (
+ "bufio"
+ "fmt"
+ "os"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var logPath string
+var (
+ repoStore database.RepoStore
+)
+
+func init() {
+ initCmd.Flags().StringVar(&logPath, "path", "", "log path of log file")
+ Cmd.AddCommand(
+ initCmd,
+ )
+}
+
+var Cmd = &cobra.Command{
+ Use: "logscan",
+ Short: "scan gitserver log to count the models and datasets downloads",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ err = fmt.Errorf("initializing DB connection: %w", err)
+ return
+ }
+ repoStore = database.NewRepoStore()
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+var initCmd = &cobra.Command{
+ Use: "gitea",
+ Short: "scan gitea log to count the models and datasets downloads",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ fmt.Println(logPath)
+ // Open log file
+ file, err := os.Open(logPath)
+ if err != nil {
+ fmt.Println("Error opening file:", err)
+ return err
+ }
+ defer file.Close()
+
+ // use regexp to find the clone log
+ // http clone log example: 2023/12/27 06:36:28 ...eb/routing/logger.go:102:func1() [I] router: completed GET /models_wayne0019/lwftest.git/info/refs?service=git-upload-pack
+ httpPattern := regexp.MustCompile(`(\d{4}\/\d{2}\/\d{2}) \d{2}:\d{2}:\d{2}.*completed GET \/(models|datasets)_([\w_-]+\/[\w_-]+)\.git\/info\/refs\?service=git-upload-pack`)
+ // ssh clone log example: 2023/12/27 06:38:04 ...eb/routing/logger.go:102:func1() [I] router: completed GET /api/internal/serv/command/15/models_zzz/test?mode=1&verb=git-upload-pack for 127.0.0.1:0, 200 OK in 3.7ms @ private/serv.go:79(private.ServCommand)
+ sshPattern := regexp.MustCompile(`(\d{4}\/\d{2}\/\d{2}) \d{2}:\d{2}:\d{2}.*completed GET \/api\/internal\/serv\/command\/\d+\/(models|datasets)_([\w_-]+\/[\w_-]+)\?mode=1&verb=git-upload-pack`)
+
+ // count clone action for each project
+ projectCount := make(map[string]map[string]map[string]int64)
+
+ // read log file line by line
+ scanner := bufio.NewScanner(file)
+ for scanner.Scan() {
+ line := scanner.Text()
+ httpMatches := httpPattern.FindStringSubmatch(line)
+ if len(httpMatches) == 4 {
+ date := httpMatches[1]
+ repoType := httpMatches[2]
+ repoPath := httpMatches[3]
+ if projectCount[date] == nil {
+ projectCount[date] = make(map[string]map[string]int64)
+ }
+
+ if projectCount[date][repoType] == nil {
+ projectCount[date][repoType] = make(map[string]int64)
+ }
+
+ projectCount[date][repoType][repoPath]++
+ }
+ sshMatches := sshPattern.FindStringSubmatch(line)
+ if len(sshMatches) == 4 {
+ date := sshMatches[1]
+ repoType := sshMatches[2]
+ repoPath := sshMatches[3]
+ if projectCount[date] == nil {
+ projectCount[date] = make(map[string]map[string]int64)
+ }
+
+ if projectCount[date][repoType] == nil {
+ projectCount[date][repoType] = make(map[string]int64)
+ }
+
+ projectCount[date][repoType][repoPath]++
+ }
+ }
+
+ for date, typeMap := range projectCount {
+ for repoTypeString, pathMap := range typeMap {
+ for path, count := range pathMap {
+ fmt.Printf("date: %s, type: %s, path: %s, count: %d\n", date, repoTypeString, path, count)
+ repoType := getRepoTypeByTypeName(repoTypeString)
+ spPath := strings.Split(path, "/")
+ repo, err := repoStore.FindByPath(cmd.Context(), repoType, spPath[0], spPath[1])
+ if err != nil {
+ fmt.Printf("Error finding %s: %s, Date: %s, Count: %d error: %v\n", repoType, path, date, count, err)
+ continue
+ }
+ pDate, _ := time.Parse("2006/01/02", date)
+ err = repoStore.UpdateRepoCloneDownloads(cmd.Context(), repo, pDate, count)
+ if err != nil {
+ fmt.Printf("Error updating %s: %s, Date: %s, Count: %d error: %v\n", repoType, path, date, count, err)
+ }
+ fmt.Printf("Update secceed %s: %s, Date: %s, Count: %d error: %v\n", repoType, path, date, count, err)
+ }
+ }
+ }
+ // check error
+ if err := scanner.Err(); err != nil {
+ fmt.Println("Error reading file:", err)
+ return err
+ }
+ return nil
+ },
+}
+
+func getRepoTypeByTypeName(repoType string) types.RepositoryType {
+ switch repoType {
+ case "datasets":
+ return types.DatasetRepo
+ case "models":
+ return types.ModelRepo
+ case "codes":
+ return types.CodeRepo
+ case "spaces":
+ return types.SpaceRepo
+ default:
+ return types.UnknownRepo
+ }
+}
+
+
+
package migration
+
+import (
+ "fmt"
+ "log/slog"
+ "os"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "github.com/uptrace/bun/migrate"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/database/migrations"
+ "opencsg.com/csghub-server/common/config"
+)
+
+// verboseMode whether to show SQL detail
+var verboseMode bool
+
+func init() {
+ Cmd.Flags().BoolVar(&verboseMode, "verbose", false, "whether to show SQL detail")
+ Cmd.AddCommand(
+ initCmd,
+ migrateCmd,
+ rollbackCmd,
+ lockCmd,
+ unlockCmd,
+ createGoCmd,
+ createSQLCmd,
+ statusCmd,
+ markAppliedCmd,
+ )
+}
+
+var (
+ migrator *migrate.Migrator
+ db *database.DB
+)
+
+var Cmd = &cobra.Command{
+ Use: "migration",
+ Short: "run database migrations",
+ Long: "migration manage database schema, keeping it up-to-date with current application logic. Developer also uses migration to create new database migration during their development.",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ if verboseMode {
+ err = os.Setenv("DB_DEBUG", "1")
+ if err != nil {
+ err = fmt.Errorf("setting ENV DB_DEBUG: %w", err)
+ return
+ }
+ }
+
+ config, err := config.LoadConfig()
+ if err != nil {
+ return
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ db, err = database.NewDB(cmd.Context(), dbConfig)
+ if err != nil {
+ err = fmt.Errorf("initializing DB connection: %w", err)
+ return
+ }
+ migrator = migrations.NewMigrator(db)
+
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+ PersistentPostRun: func(cmd *cobra.Command, args []string) {
+ if db != nil {
+ _ = db.Close()
+ }
+ },
+}
+
+var initCmd = &cobra.Command{
+ Use: "init",
+ Short: "create migration tables",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return migrator.Init(cmd.Context())
+ },
+}
+
+var migrateCmd = &cobra.Command{
+ Use: "migrate",
+ Short: "migrate database",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ group, err := migrator.Migrate(cmd.Context())
+ if err != nil {
+ return err
+ }
+ if group.IsZero() {
+ slog.Info("there are no new migrations to run (database is up to date)")
+ return nil
+ }
+ slog.Info(fmt.Sprintf("migrated to %s", group))
+ return nil
+ },
+}
+
+var rollbackCmd = &cobra.Command{
+ Use: "rollback",
+ Short: "rollback the last migration group",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ group, err := migrator.Rollback(cmd.Context())
+ if err != nil {
+ return err
+ }
+ if group.IsZero() {
+ slog.Info("there are no groups to roll back")
+ return nil
+ }
+ slog.Info(fmt.Sprintf("rolled back %s", group))
+ return nil
+ },
+}
+
+var lockCmd = &cobra.Command{
+ Use: "lock",
+ Short: "lock migrations",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return migrator.Lock(cmd.Context())
+ },
+}
+
+var unlockCmd = &cobra.Command{
+ Use: "unlock",
+ Short: "unlock migrations",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return migrator.Unlock(cmd.Context())
+ },
+}
+
+var createGoCmd = &cobra.Command{
+ Use: "create_go",
+ Short: "create Go migration for developers",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ name := strings.Join(args, "_")
+ mf, err := migrator.CreateGoMigration(cmd.Context(), name)
+ if err != nil {
+ return err
+ }
+ slog.Info("created migration %s (%s)", mf.Name, mf.Path)
+ return nil
+ },
+}
+
+var createSQLCmd = &cobra.Command{
+ Use: "create_sql",
+ Short: "create up and down SQL migrations for developers",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ name := strings.Join(args, "_")
+ files, err := migrator.CreateSQLMigrations(cmd.Context(), name)
+ if err != nil {
+ return err
+ }
+
+ for _, mf := range files {
+ slog.Info("created migration %s (%s)", mf.Name, mf.Path)
+ }
+
+ return nil
+ },
+}
+
+var statusCmd = &cobra.Command{
+ Use: "status",
+ Short: "print migrations status",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ ms, err := migrator.MigrationsWithStatus(cmd.Context())
+ if err != nil {
+ return err
+ }
+ slog.Info(fmt.Sprintf("migrations: %s", ms))
+ slog.Info(fmt.Sprintf("unapplied migrations: %s", ms.Unapplied()))
+ slog.Info(fmt.Sprintf("last migration group: %s", ms.LastGroup()))
+ return nil
+ },
+}
+
+var markAppliedCmd = &cobra.Command{
+ Use: "mark_applied",
+ Short: "mark migrations as applied without actually running them",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ group, err := migrator.Migrate(cmd.Context(), migrate.WithNopMigration())
+ if err != nil {
+ return err
+ }
+ if group.IsZero() {
+ slog.Info("there are no new migrations to mark as applied")
+ return nil
+ }
+ slog.Info(fmt.Sprintf("marked as applied %s", group))
+ return nil
+ },
+}
+
+
+
package mirror
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/cache"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+const (
+ resourceName = "check-mirror-progress"
+ expirationTime = 1 * time.Hour
+)
+
+var resync bool
+
+func init() {
+ checkMirrorProgress.Flags().BoolVar(&resync, "resync", false, "the path of the file")
+}
+
+var checkMirrorProgress = &cobra.Command{
+ Use: "check-mirror-progress",
+ Short: "the cmd to check mirror progress",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+
+ if config.GitServer.Type != types.GitServerTypeGitea {
+ return
+ }
+
+ locker, err := cache.NewCache(ctx, cache.RedisConfig{
+ Addr: config.Redis.Endpoint,
+ Username: config.Redis.User,
+ Password: config.Redis.Password,
+ })
+
+ if err != nil {
+ slog.Error("failed to initialize redis", "err", err)
+ return
+ }
+
+ err = locker.RunWhileLocked(ctx, resourceName, expirationTime, func(ctx context.Context) error {
+ c, err := component.NewMirrorComponent(config)
+ if err != nil {
+ slog.Error("failed to create mirror component", "err", err)
+ return err
+ }
+
+ err = c.CheckMirrorProgress(ctx)
+ if err != nil {
+ slog.Error("failed to check mirror progress", "err", err)
+ return err
+ }
+ return nil
+ })
+ if err != nil {
+ slog.Error("failed to check mirror progress", "err", err)
+ return
+ }
+
+ },
+}
+
+
+
package mirror
+
+import (
+ "bufio"
+ "context"
+ "encoding/csv"
+ "fmt"
+ "log/slog"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+var (
+ filePath string
+ lfs bool
+)
+
+func init() {
+ createMirrorRepoFromFile.Flags().StringVar(&filePath, "file", "", "the path of the file")
+ createMirrorRepoFromFile.Flags().BoolVar(&lfs, "lfs", false, "sync lfs file")
+}
+
+var createMirrorRepoFromFile = &cobra.Command{
+ Use: "create-mirror-from-file",
+ Short: "the cmd to create mirror repository from file",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ if filePath == "" {
+ return fmt.Errorf("empty file path")
+ }
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+ if !config.Saas {
+ return
+ }
+
+ c, err := component.NewMirrorComponent(config)
+ if err != nil {
+ slog.Error("failed to create mirror component", "err", err)
+ return
+ }
+ mirrorSourceStore := database.NewMirrorSourceStore()
+ mirrorSource, err := mirrorSourceStore.FindByName(ctx, "huggingface")
+ if err != nil {
+ slog.Error("failed to find mirror source, Please create mirror source first", "err", err)
+ return
+ }
+
+ absPath, err := filepath.Abs(filePath)
+ if err != nil {
+ slog.Error("error getting absolute path:", "err", err)
+ return
+ }
+ file, err := os.Open(absPath)
+ if err != nil {
+ slog.Error("error opening file:", "err", err)
+ return
+ }
+ defer file.Close()
+
+ reader := csv.NewReader(bufio.NewReader(file))
+ reader.TrimLeadingSpace = true
+
+ if _, err := reader.Read(); err != nil {
+ slog.Error("error reading header:", "err", err)
+ return
+ }
+
+ for {
+ record, err := reader.Read()
+ if err != nil {
+ slog.Error("error reading data:", "err", err)
+ break
+ }
+
+ var sourceGitCloneUrl string
+
+ repoType := record[4]
+ repoPath := record[1]
+ sourceNamespace := strings.Split(repoPath, "/")[0]
+ sourceName := strings.Split(repoPath, "/")[1]
+ if repoType == "model" {
+ sourceGitCloneUrl = fmt.Sprintf("https://huggingface.co/%s", repoPath)
+ } else {
+ sourceGitCloneUrl = fmt.Sprintf("https://huggingface.co/%s", fmt.Sprintf("%ss/%s", repoType, repoPath))
+ }
+
+ req := types.CreateMirrorRepoReq{
+ SourceNamespace: sourceNamespace,
+ SourceName: sourceName,
+ MirrorSourceID: mirrorSource.ID,
+ RepoType: types.RepositoryType(repoType),
+ DefaultBranch: "main",
+ SourceGitCloneUrl: sourceGitCloneUrl,
+ SyncLfs: lfs,
+ }
+ fmt.Println(req)
+ _, err = c.CreateMirrorRepo(ctx, req)
+ if err != nil {
+ slog.Error("error creating mirror:", "err", err)
+ continue
+ }
+ slog.Info("create mirror successfully", slog.String("source_url", sourceGitCloneUrl))
+ }
+ },
+}
+
+
+
package mirror
+
+import (
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/mirror"
+)
+
+var lfsSyncCmd = &cobra.Command{
+ Use: "lfs-sync",
+ Short: "Start the repoisotry lfs files sync server",
+ Example: lfsSyncExample(),
+ RunE: func(*cobra.Command, []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+
+ lfsSyncWorker, err := mirror.NewLFSSyncWorker(cfg, cfg.Mirror.WorkerNumber)
+ if err != nil {
+ return err
+ }
+ lfsSyncWorker.Run()
+
+ return nil
+ },
+}
+
+func lfsSyncExample() string {
+ return `
+# for development
+csghub-server mirror lfs-sync
+`
+}
+
+
+
package mirror
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(createMirrorRepoFromFile)
+ Cmd.AddCommand(checkMirrorProgress)
+ Cmd.AddCommand(lfsSyncCmd)
+ Cmd.AddCommand(repoSyncCmd)
+}
+
+var Cmd = &cobra.Command{
+ Use: "mirror",
+ Short: "entry point for mirror jobs",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package mirror
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/workflow"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/mirror"
+)
+
+var repoSyncCmd = &cobra.Command{
+ Use: "repo-sync",
+ Short: "Start the repoisotry sync server",
+ Example: repoSyncExample(),
+ RunE: func(*cobra.Command, []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+
+ err = workflow.StartWorker(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to start worker: %w", err)
+ }
+
+ repoSYncer, err := mirror.NewRepoSyncWorker(cfg, cfg.Mirror.WorkerNumber)
+ if err != nil {
+ return err
+ }
+
+ repoSYncer.Run()
+
+ workflow.StopWorker()
+
+ return nil
+ },
+}
+
+func repoSyncExample() string {
+ return `
+# for development
+csghub-server mirror repo-sync
+`
+}
+
+
+
package moderation
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/moderation/checker"
+ "opencsg.com/csghub-server/moderation/router"
+ "opencsg.com/csghub-server/moderation/workflow"
+)
+
+var cmdLaunch = &cobra.Command{
+ Use: "launch",
+ Short: "Launch moderation server",
+ Example: serverExample(),
+ RunE: func(cmd *cobra.Command, args []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ slog.Debug("config", slog.Any("data", cfg))
+ // Check APIToken length
+ if len(cfg.APIToken) < 128 {
+ return fmt.Errorf("API token length is less than 128, please check")
+ }
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+ checker.Init(cfg)
+
+ //init async moderation process
+ err = workflow.StartWorker(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to start workflow worker,%w", err)
+ }
+
+ r, err := router.NewRouter(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+ slog.Info("moderation http server is running", slog.Any("port", cfg.Moderation.Port))
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.Moderation.Port,
+ },
+ r,
+ )
+ server.Run()
+
+ workflow.StopWorker()
+
+ return nil
+ },
+}
+
+func serverExample() string {
+ return `
+# for development
+csghub-server moderation launch
+`
+}
+
+
+
package moderation
+
+import "github.com/spf13/cobra"
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(cmdLaunch)
+}
+
+var Cmd = &cobra.Command{
+ Use: "moderation",
+ Short: "entry point for moderation service",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package payment
+
+import (
+ "fmt"
+ "github.com/spf13/cobra"
+ "log/slog"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/payment/router"
+)
+
+var launchCmd = &cobra.Command{
+ Use: "launch",
+ Short: "Launch payment server",
+ Example: serverExample(),
+ RunE: func(cmd *cobra.Command, args []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ slog.Debug("config", slog.Any("data", cfg))
+ // Check APIToken length
+ if len(cfg.APIToken) < 128 {
+ return fmt.Errorf("API token length is less than 128, please check")
+ }
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+
+ err = event.InitEventPublisher(cfg, nil)
+ if err != nil {
+ return fmt.Errorf("fail to initialize message queue, %w", err)
+ }
+
+ r, err := router.NewPaymentRouter(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+ slog.Info("http server is running", slog.Any("port", cfg.Payment.Port))
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.Payment.Port,
+ },
+ r,
+ )
+ server.Run()
+
+ return nil
+ },
+}
+
+func serverExample() string {
+ return `
+# for development
+csghub-server payment launch
+`
+}
+
+
+
package payment
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(launchCmd)
+}
+
+var Cmd = &cobra.Command{
+ Use: "payment",
+ Short: "entry point for payment service",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package cmd
+
+import (
+ "fmt"
+ "log/slog"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/payment"
+ "os"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/accounting"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/cron"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/deploy"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/git"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/logscan"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/migration"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/mirror"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/moderation"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/start"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/sync"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/trigger"
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd/user"
+ "opencsg.com/csghub-server/common/config"
+)
+
+var (
+ logLevel string
+ logFormat string
+ configFile string
+)
+
+var RootCmd = &cobra.Command{
+ Use: "csghub-server",
+ Short: "Back-end API server for starhub.",
+ SilenceUsage: true,
+}
+
+func init() {
+ var err error
+ defer func() {
+ if err != nil {
+ panic(err)
+ }
+ }()
+
+ RootCmd.PersistentFlags().StringVarP(&logLevel, "log-level", "l", "info", "set log level to debug, info, warn, error or fatal (case-insensitive). default is INFO")
+ RootCmd.PersistentFlags().StringVarP(&logFormat, "log-format", "f", "json", "set log format to json or text. default is json")
+ RootCmd.PersistentFlags().StringVarP(&configFile, "config", "", "", "set config file path.")
+ RootCmd.DisableAutoGenTag = true
+
+ cobra.OnInitialize(func() {
+ setupLog(logLevel, logFormat)
+ config.SetConfigFile(configFile)
+ })
+
+ RootCmd.AddCommand(
+ migration.Cmd,
+ start.Cmd,
+ logscan.Cmd,
+ trigger.Cmd,
+ deploy.Cmd,
+ cron.Cmd,
+ mirror.Cmd,
+ accounting.Cmd,
+ sync.Cmd,
+ user.Cmd,
+ git.Cmd,
+ moderation.Cmd,
+ payment.Cmd,
+ )
+}
+
+func setupLog(lvl, format string) {
+ logLevel := slog.LevelInfo.Level()
+ var logger *slog.Logger
+ if len(lvl) > 0 {
+ err := logLevel.UnmarshalText([]byte(lvl))
+ // logLevel not change if unmarshall failed
+ if err != nil {
+ fmt.Println("input invalid log level, use default log level INFO")
+ }
+ }
+ // TODO:log source file position
+ opt := &slog.HandlerOptions{AddSource: false, Level: logLevel}
+ var handler slog.Handler
+ switch format {
+ case "json":
+ handler = slog.NewJSONHandler(os.Stdout, opt)
+ default:
+ handler = slog.NewTextHandler(os.Stdout, opt)
+ }
+ fmt.Printf("init logger, level: %s, format: %s\n", logLevel.String(), format)
+ logger = slog.New(handler)
+ slog.SetDefault(logger)
+}
+
+
+
package start
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/api/router"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+)
+
+var rproxyCmd = &cobra.Command{
+ Use: "rproxy",
+ Short: "Start the reverse proxy server",
+ Example: rproxyExample(),
+ RunE: func(*cobra.Command, []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+ r, err := router.NewRProxyRouter(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.Space.RProxyServerPort,
+ },
+ r,
+ )
+ server.Run()
+
+ return nil
+ },
+}
+
+func rproxyExample() string {
+ return `
+# for development
+csghub-server start rproxy
+`
+}
+
+
+
package start
+
+import (
+ "fmt"
+ "net/url"
+ "time"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/api/router"
+ "opencsg.com/csghub-server/api/workflow"
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/docs"
+)
+
+var enableSwagger bool
+
+func init() {
+ serverCmd.Flags().BoolVar(&enableSwagger, "swagger", false, "Start swagger help docs")
+}
+
+var serverCmd = &cobra.Command{
+ Use: "server",
+ Short: "Start the API server",
+ Example: serverExample(),
+ RunE: func(cmd *cobra.Command, args []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+
+ enableSwagger = enableSwagger || cfg.EnableSwagger
+
+ if enableSwagger {
+ // @securityDefinitions.apikey ApiKey
+ // @in header
+ // @name Authorization
+ // @description Bearer token
+ publicDomain, err := url.Parse(cfg.APIServer.PublicDomain)
+ if err != nil {
+ return fmt.Errorf("failed to parse api server public domain: %v", err)
+ }
+ docs.SwaggerInfo.Title = "CSGHub Server API"
+ docs.SwaggerInfo.Description = "CSGHub Server API."
+ docs.SwaggerInfo.Version = "1.0"
+ docs.SwaggerInfo.Host = publicDomain.Host
+ docs.SwaggerInfo.BasePath = "/api/v1"
+ docs.SwaggerInfo.Schemes = []string{"http", "https"}
+ }
+
+ // Check APIToken length
+ if len(cfg.APIToken) < 128 {
+ return fmt.Errorf("API token length is less than 128, please check")
+ }
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+ err = event.InitEventPublisher(cfg, nil)
+ if err != nil {
+ return fmt.Errorf("fail to initialize message queue, %w", err)
+ }
+ err = deploy.Init(deploy.DeployConfig{
+ ImageBuilderURL: cfg.Space.BuilderEndpoint,
+ ImageRunnerURL: cfg.Space.RunnerEndpoint,
+ MonitorInterval: 10 * time.Second,
+ InternalRootDomain: cfg.Space.InternalRootDomain,
+ SpaceDeployTimeoutInMin: cfg.Space.DeployTimeoutInMin,
+ ModelDeployTimeoutInMin: cfg.Model.DeployTimeoutInMin,
+ ModelDownloadEndpoint: cfg.Model.DownloadEndpoint,
+ ChargingEnable: cfg.Accounting.ChargingEnable,
+ AutoCleanInstance: cfg.AutoClean.Instance,
+ PublicRootDomain: cfg.Space.PublicRootDomain,
+ IsMasterHost: cfg.IsMasterHost,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to init deploy: %w", err)
+ }
+
+ r, err := router.NewRouter(cfg, enableSwagger)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+
+ err = workflow.StartWorker(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to start worker: %w", err)
+ }
+
+ err = workflow.RegisterCronJobs(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to register cron jobs: %w", err)
+ }
+
+ err = workflow.StartCronWorker(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to start cron worker: %w", err)
+ }
+
+ // Initialize mirror service
+ // mirrorService, err := mirror.NewMirrorPriorityQueue(cfg)
+ // if err != nil {
+ // return fmt.Errorf("failed to init mirror service: %w", err)
+ // }
+
+ // if cfg.MirrorServer.Enable && cfg.GitServer.Type == types.GitServerTypeGitaly {
+ // go mirrorService.EnqueueMirrorTasks()
+ // }
+
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.APIServer.Port,
+ },
+ r,
+ )
+ server.Run()
+ workflow.StopWorker()
+
+ return nil
+ },
+}
+
+func serverExample() string {
+ return `
+# for development
+csghub-server start server
+`
+}
+
+
+
package start
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/database/migrations"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func init() {
+ Cmd.AddCommand(serverCmd)
+ Cmd.AddCommand(rproxyCmd)
+}
+
+var Cmd = &cobra.Command{
+ Use: "start",
+ Short: "Start a service",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ // Ensure database schema is up-to-date
+ ctx, cancel := context.WithTimeout(cmd.Context(), 10*time.Second)
+ defer cancel()
+
+ config, err := config.LoadConfig()
+ if err != nil {
+ return
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ db, err := database.NewDB(cmd.Context(), dbConfig)
+ if err != nil {
+ err = fmt.Errorf("initializing DB connection: %w", err)
+ return
+ }
+ migrator := migrations.NewMigrator(db)
+
+ status, err := migrator.MigrationsWithStatus(ctx)
+ if err != nil {
+ err = fmt.Errorf("listing database migration status: %w", err)
+ return
+ }
+
+ unapplied := status.Unapplied()
+ if len(unapplied) > 0 {
+ err = fmt.Errorf("abort to start, there are unapplied database migrations: %s", unapplied)
+ return
+ }
+
+ return
+ },
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return cmd.Help()
+ },
+}
+
+
+
package sync
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/multisync"
+ "opencsg.com/csghub-server/builder/store/cache"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/component"
+)
+
+const (
+ resourceName = "sync-as-client"
+ expirationTime = 1 * time.Hour
+)
+
+var cmdSyncAsClient = &cobra.Command{
+ Use: "sync-as-client",
+ Short: "the cmd to sync repos like models and datasets from remote server like OpenCSG",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+
+ if config.Saas {
+ return
+ }
+
+ locker, err := cache.NewCache(ctx, cache.RedisConfig{
+ Addr: config.Redis.Endpoint,
+ Username: config.Redis.User,
+ Password: config.Redis.Password,
+ })
+
+ if err != nil {
+ slog.Error("failed to initialize redis", "err", err)
+ return
+ }
+
+ err = locker.RunWhileLocked(ctx, resourceName, expirationTime, func(ctx context.Context) error {
+ c, err := component.NewMultiSyncComponent(config)
+ if err != nil {
+ slog.Error("failed to create multi sync component", "err", err)
+ return err
+ }
+ syncClientSettingStore := database.NewSyncClientSettingStore()
+ setting, err := syncClientSettingStore.First(ctx)
+ if err != nil {
+ slog.Error("failed to find sync client setting", "error", err)
+ return err
+ }
+ apiDomain := config.MultiSync.SaasAPIDomain
+ sc := multisync.FromOpenCSG(apiDomain, setting.Token)
+ return c.SyncAsClient(ctx, sc)
+ })
+ if err != nil {
+ slog.Error("failed to sync as client", "err", err)
+ }
+ },
+}
+
+
+
package sync
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/multisync"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var cmdFixDefaultBranch = &cobra.Command{
+ Use: "fix-default-branch",
+ Short: "the cmd to sync repos like models and datasets from remote server like OpenCSG",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+
+ if config.Saas {
+ return
+ }
+
+ if !config.MultiSync.Enabled {
+ return
+ }
+
+ repoStore := database.NewRepoStore()
+ var (
+ batch = 0
+ batchSize = 1000
+ )
+ for {
+ repositories, err := repoStore.FindByRepoSourceWithBatch(ctx, types.OpenCSGSource, batchSize, batch)
+ if err != nil {
+ slog.Error("failed to find repositories from OpenCSG", "error", err)
+ return
+ }
+ if len(repositories) == 0 {
+ slog.Info("no more repositories found from OpenCSG, quit")
+ return
+ }
+ syncClientSettingStore := database.NewSyncClientSettingStore()
+ setting, err := syncClientSettingStore.First(ctx)
+ if err != nil {
+ slog.Error("failed to find sync client setting", "error", err)
+ return
+ }
+ apiDomain := config.MultiSync.SaasAPIDomain
+ sc := multisync.FromOpenCSG(apiDomain, setting.Token)
+ for _, repository := range repositories {
+ var defaultBranch string
+ repoPath := strings.TrimPrefix(repository.Path, types.OpenCSGPrefix)
+ if repository.RepositoryType == types.ModelRepo {
+ modelInfo, err := sc.ModelInfo(ctx, types.SyncVersion{RepoPath: repoPath})
+ if err != nil {
+ slog.Error("failed to get model info from OpenCSG Saas", slog.String("repo_path", repoPath), slog.Any("error", err))
+ continue
+ }
+ defaultBranch = modelInfo.DefaultBranch
+ } else if repository.RepositoryType == types.DatasetRepo {
+ datasetInfo, err := sc.DatasetInfo(ctx, types.SyncVersion{RepoPath: repoPath})
+ if err != nil {
+ slog.Error("failed to get dataset info from OpenCSG Saas", slog.String("repo_path", repoPath), slog.Any("error", err))
+ continue
+ }
+ defaultBranch = datasetInfo.DefaultBranch
+ }
+ repository.DefaultBranch = defaultBranch
+ _, err = repoStore.UpdateRepo(ctx, repository)
+ if err != nil {
+ slog.Error("failed to update repository", slog.String("repo_path", repoPath), slog.Any("error", err))
+ continue
+ }
+ }
+ batch += 1
+ }
+ },
+}
+
+
+
package sync
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+var InitCmd = &cobra.Command{
+ Use: "init",
+ Short: "init syncversion table",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config,%w", err)
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ return fmt.Errorf("initializing DB connection: %w", err)
+ }
+ ctx := context.WithValue(cmd.Context(), "config", config)
+ cmd.SetContext(ctx)
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ ctx := cmd.Context()
+ config, ok := ctx.Value("config").(*config.Config)
+ if !ok {
+ slog.Error("config not found in context")
+ return
+ }
+ var versions []database.SyncVersion
+ repoComponent, err := component.NewRepoComponent(config)
+ if err != nil {
+ slog.Error("failed to create repository component", "error", err)
+ return
+ }
+ mirrorRepo := database.NewMirrorStore()
+ syncVersionStore := database.NewSyncVersionStore()
+
+ mirrors, err := mirrorRepo.Finished(ctx)
+ if err != nil {
+ slog.Error("error finding mirror repositories", "error", err)
+ return
+ }
+ for _, mirror := range mirrors {
+ repo := mirror.Repository
+ if repo == nil {
+ continue
+ }
+ if repo.Private {
+ continue
+ }
+ namespace, name := repo.NamespaceAndName()
+ req := &types.GetCommitsReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: repo.DefaultBranch,
+ RepoType: repo.RepositoryType,
+ }
+ commit, err := repoComponent.LastCommit(ctx, req)
+ if err != nil {
+ slog.Error("error getting repository last commit", "error", err)
+ continue
+ }
+
+ versions = append(versions, database.SyncVersion{
+ SourceID: types.SyncVersionSourceOpenCSG,
+ RepoPath: repo.Path,
+ RepoType: repo.RepositoryType,
+ LastModifiedAt: repo.UpdatedAt,
+ ChangeLog: commit.Message,
+ })
+ }
+ if len(versions) == 0 {
+ slog.Error("there are no finished mirror repositories")
+ return
+ }
+ err = syncVersionStore.BatchCreate(ctx, versions)
+ if err != nil {
+ slog.Error("failed to init sync version error", "error", err)
+ return
+ }
+ slog.Info("sync versions successfully")
+ },
+}
+
+
+
package sync
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/multisync/router"
+)
+
+var syncServerCmd = &cobra.Command{
+ Use: "sync-server",
+ Short: "Start the multi source sync server",
+ Example: syncServerExample(),
+ RunE: func(*cobra.Command, []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+ r, err := router.NewRouter(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.Mirror.Port,
+ },
+ r,
+ )
+ server.Run()
+
+ return nil
+ },
+}
+
+func syncServerExample() string {
+ return `
+# for development
+csghub-server sync sync-server
+`
+}
+
+
+
package sync
+
+import (
+ "github.com/spf13/cobra"
+)
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(InitCmd)
+ Cmd.AddCommand(cmdSyncAsClient)
+ Cmd.AddCommand(syncServerCmd)
+ Cmd.AddCommand(cmdFixDefaultBranch)
+}
+
+var Cmd = &cobra.Command{
+ Use: "sync",
+ Short: "entry point for mirror jobs",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package trigger
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "os"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/user/component"
+)
+
+var fixOrgDataCmd = &cobra.Command{
+ Use: "fix-org-data",
+ Short: "scan organization and fix organization data, like init read,write and admin role for org",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ lh := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+ AddSource: false,
+ Level: slog.LevelInfo,
+ })
+ l := slog.New(lh)
+ slog.SetDefault(l)
+
+ var orgs []database.Organization
+ var err error
+ var cfg *config.Config
+ cfg, err = config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ ctx := context.Background()
+ os := database.NewOrgStore()
+ orgComponent, _ := component.NewOrganizationComponent(cfg)
+
+ // get all organizations
+ orgs, _, err = os.GetUserOwnOrgs(ctx, "")
+ for _, org := range orgs {
+ req := new(types.CreateOrgReq)
+ req.Name = org.Name
+ req.Nickname = org.Nickname
+ req.Username = org.User.Username
+ req.Description = org.Description
+
+ slog.Info("before create org", slog.Any("req", req))
+ _, err1 := orgComponent.FixOrgData(ctx, &org)
+ if err1 != nil {
+ err = errors.Join(err, err1)
+ slog.Error("create org has error", slog.String("error", err.Error()))
+ }
+ slog.Info("done create org", slog.String("org", req.Name))
+ }
+ return err
+ },
+}
+
+
+
package trigger
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "os"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/component"
+)
+
+var fixUserDataCmd = &cobra.Command{
+ Use: "fix-user-data",
+ Short: "scan user and fix user data",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ lh := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
+ AddSource: false,
+ Level: slog.LevelInfo,
+ })
+ l := slog.New(lh)
+ slog.SetDefault(l)
+
+ var users []database.User
+ var err error
+ var cfg *config.Config
+ cfg, err = config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ ctx := context.Background()
+ userStore := database.NewUserStore()
+ userComponent, _ := component.NewUserComponent(cfg)
+
+ // get all organizations
+ users, err = userStore.Index(ctx)
+ for _, user := range users {
+ err1 := userComponent.FixUserData(ctx, user.Username)
+ if err1 != nil {
+ err = errors.Join(err, err1)
+ slog.Error("create user's orgs has error", slog.String("error", err.Error()))
+ }
+ slog.Info("done create user's orgs", slog.String("org", user.Username))
+ }
+ return err
+ },
+}
+
+
+
package trigger
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "github.com/spf13/cobra"
+ "go.temporal.io/sdk/client"
+ "opencsg.com/csghub-server/api/workflow"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var (
+ // callbackComponent *callback.GitCallbackComponent
+ rs database.RepoStore
+ gs gitserver.GitServer
+
+ repoPaths []string
+)
+
+func init() {
+ repoPaths = make([]string, 0)
+ gitCallbackCmd.Flags().StringSliceVar(&repoPaths, "repos", nil,
+ "paths of repositories to trigger callback, path in format '[repo_type]/[owner]/[repo_name]', for example 'datasets/leida/stg-test-dataset,models/leida/stg-test-model'")
+}
+
+var gitCallbackCmd = &cobra.Command{
+ Use: "gitcallback",
+ Short: "scan repository and trigger callback for meta tags re-processing",
+ RunE: func(cmd *cobra.Command, args []string) error {
+ var err error
+ var repos []*database.Repository
+ config, err := config.LoadConfig()
+ if err != nil {
+ return fmt.Errorf("failed to load config: %w", err)
+ }
+ err = workflow.StartWorker(config)
+ if err != nil {
+ slog.Error("failed to start worker", slog.Any("error", err))
+ return fmt.Errorf("failed to start worker: %w", err)
+ }
+ if len(repoPaths) > 0 {
+ for _, rp := range repoPaths {
+ parts := strings.Split(rp, "/")
+ if len(parts) != 3 {
+ slog.Error(fmt.Sprintf("invalid repo path, skip: %s", rp))
+ continue
+ }
+ repoType := parts[0]
+ owner := parts[1]
+ repoName := parts[2]
+ repo, err := rs.Find(context.Background(), owner, repoType, repoName)
+ if err != nil {
+ slog.Error("fail to find repository, skip", slog.String("repo", rp), slog.Any("error", err))
+ continue
+ }
+ repos = append(repos, repo)
+ }
+ } else {
+ repos, err = rs.All(context.Background())
+ }
+
+ if err != nil {
+ return err
+ }
+ for _, repo := range repos {
+ splits := strings.Split(repo.Path, "/")
+ namespace, repoName := splits[0], splits[1]
+ req := &types.GiteaCallbackPushReq{}
+ var err error
+ // file paths relative to repository root
+ var filePaths []string
+ filePaths, err = getFilePaths(namespace, repoName, "", repo.RepositoryType, gs.GetRepoFileTree)
+ if err != nil {
+ slog.Error("failed to get file names", slog.String("repo", repo.Path), slog.Any("error", err))
+ continue
+ }
+ slog.Info("file names from git server ", "fileNames", filePaths)
+ req.Repository.FullName = repo.GitPath
+ req.Commits = append(req.Commits, types.GiteaCallbackPushReq_Commit{})
+ req.Commits[0].Added = append(req.Commits[0].Added, filePaths...)
+
+ //start workflow to handle push request
+ workflowClient := workflow.GetWorkflowClient()
+ workflowOptions := client.StartWorkflowOptions{
+ TaskQueue: workflow.HandlePushQueueName,
+ }
+
+ we, err := workflowClient.ExecuteWorkflow(context.Background(), workflowOptions, workflow.HandlePushWorkflow,
+ req,
+ config,
+ )
+ if err != nil {
+ slog.Error("failed to handle git push callback", slog.String("repo", repo.Path), slog.Any("error", err))
+ return err
+ }
+
+ slog.Info("start handle push workflow", slog.String("workflow_id", we.GetID()), slog.String("run_id", we.GetRunID()), slog.Any("req", &req))
+ slog.Info("trigger complete", slog.String("repo", repo.Path), slog.String("type", string(repo.RepositoryType)), slog.Any("error", err))
+ }
+ return nil
+ },
+}
+
+func getFilePaths(namespace, repoName, folder string, repoType types.RepositoryType, gsTree func(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error)) ([]string, error) {
+ var filePaths []string
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: repoName,
+ Ref: "",
+ Path: folder,
+ RepoType: repoType,
+ }
+ gitFiles, err := gsTree(context.Background(), getRepoFileTree)
+ if err != nil {
+ slog.Error("Failed to get dataset file contents", slog.String("path", folder), slog.Any("error", err))
+ return filePaths, err
+ }
+ for _, file := range gitFiles {
+ if file.Type == "dir" {
+ subFileNames, err := getFilePaths(namespace, repoName, file.Path, repoType, gsTree)
+ if err != nil {
+ return filePaths, err
+ }
+ filePaths = append(filePaths, subFileNames...)
+ } else {
+ filePaths = append(filePaths, file.Path)
+ }
+ }
+ return filePaths, nil
+}
+
+
+
package trigger
+
+import (
+ "fmt"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+)
+
+func init() {
+ Cmd.AddCommand(
+ gitCallbackCmd,
+ fixOrgDataCmd,
+ fixUserDataCmd,
+ updateRepoCmd,
+ )
+}
+
+var Cmd = &cobra.Command{
+ Use: "trigger",
+ Short: "trigger a specific command",
+ PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ return
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ err = fmt.Errorf("initializing DB connection: %w", err)
+ return
+ }
+ rs = database.NewRepoStore()
+ gs, err = git.NewGitServer(config)
+ if err != nil {
+ return
+ }
+
+ return
+ },
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package trigger
+
+import (
+ "context"
+ "log/slog"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type UpdateRepoCmdArg struct {
+ RepoIDs []int64
+}
+
+var updateRepoCmdArg UpdateRepoCmdArg
+
+func init() {
+ updateRepoCmd.Flags().Int64SliceVar(&updateRepoCmdArg.RepoIDs, "repo-ids", []int64{}, "list of repo ids")
+}
+
+var updateRepoCmd = &cobra.Command{
+ Use: "update-repo",
+ Short: "update repo",
+ Run: func(cmd *cobra.Command, args []string) {
+ config, err := config.LoadConfig()
+ if err != nil {
+ slog.Error("failed to load config", "err", err)
+ return
+ }
+
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(config.Database.Driver),
+ DSN: config.Database.DSN,
+ }
+
+ database.InitDB(dbConfig)
+ if err != nil {
+ slog.Error("initializing DB connection", "err", err)
+ return
+ }
+
+ if !config.Saas {
+ slog.Info("saas mode, skip sensitive check")
+ return
+ }
+
+ ctx := context.Background()
+ repoStore := database.NewRepoStore()
+ for _, repoID := range updateRepoCmdArg.RepoIDs {
+ repo, err := repoStore.FindById(ctx, repoID)
+ if err != nil {
+ slog.Error("failed to get repo", "err", err, "repo_id", repoID)
+ continue
+ }
+ repo.Private = false
+ repo.SensitiveCheckStatus = types.SensitiveCheckPass
+ _, err = repoStore.UpdateRepo(ctx, *repo)
+ if err != nil {
+ slog.Error("failed to update repo", "err", err, "repo_id", repo.ID)
+ continue
+ }
+ slog.Info("update repo success", "repo_id", repo.ID)
+ }
+ },
+}
+
+
+
package user
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/spf13/cobra"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/router"
+ "opencsg.com/csghub-server/user/workflow"
+)
+
+var cmdLaunch = &cobra.Command{
+ Use: "launch",
+ Short: "Launch user server",
+ Example: serverExample(),
+ RunE: func(cmd *cobra.Command, args []string) (err error) {
+ cfg, err := config.LoadConfig()
+ if err != nil {
+ return err
+ }
+ slog.Debug("config", slog.Any("data", cfg))
+ // Check APIToken length
+ if len(cfg.APIToken) < 128 {
+ return fmt.Errorf("API token length is less than 128, please check")
+ }
+ dbConfig := database.DBConfig{
+ Dialect: database.DatabaseDialect(cfg.Database.Driver),
+ DSN: cfg.Database.DSN,
+ }
+ database.InitDB(dbConfig)
+
+ err = workflow.StartWorker(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to start user workflow worker: %w", err)
+ }
+
+ r, err := router.NewRouter(cfg)
+ if err != nil {
+ return fmt.Errorf("failed to init router: %w", err)
+ }
+ slog.Info("http server is running", slog.Any("port", cfg.User.Port))
+ server := httpbase.NewGracefulServer(
+ httpbase.GraceServerOpt{
+ Port: cfg.User.Port,
+ },
+ r,
+ )
+ server.Run()
+
+ workflow.StopWorker()
+
+ return nil
+ },
+}
+
+func serverExample() string {
+ return `
+# for development
+csghub-server user launch
+`
+}
+
+
+
package user
+
+import "github.com/spf13/cobra"
+
+func init() {
+ // add subcommands here
+ Cmd.AddCommand(cmdLaunch)
+}
+
+var Cmd = &cobra.Command{
+ Use: "user",
+ Short: "entry point for user service",
+ Run: func(cmd *cobra.Command, args []string) {
+ _ = cmd.Help()
+ },
+}
+
+
+
package main
+
+import (
+ "context"
+ "os"
+
+ "opencsg.com/csghub-server/cmd/csghub-server/cmd"
+)
+
+func main() {
+ command := cmd.RootCmd
+ if err := command.ExecuteContext(context.Background()); err != nil {
+ os.Exit(1)
+ }
+}
+
+
+
package config
+
+import (
+ "context"
+ "os"
+
+ "github.com/naoina/toml"
+ "github.com/sethvargo/go-envconfig"
+)
+
+var configFile = ""
+
+type Config struct {
+ Saas bool `env:"STARHUB_SERVER_SAAS, default=false"`
+ InstanceID string `env:"STARHUB_SERVER_INSTANCE_ID"`
+ EnableSwagger bool `env:"STARHUB_SERVER_ENABLE_SWAGGER, default=false"`
+ // enable if you want to acess csghub through https, especially for space rproxy
+ EnableHTTPS bool `env:"STARHUB_SERVER_ENABLE_HTTPS, default=false"`
+ APIToken string `env:"STARHUB_SERVER_API_TOKEN, default=f3a7b9c1d6e5f8e2a1b5d4f9e6a2b8d7c3a4e2b1d9f6e7a8d2c5a7b4c1e3f5b8a1d4f9b7d6e2f8a5d3b1e7f9c6a8b2d1e4f7d5b6e9f2a4b3c8e1d7f995hd82hf"`
+ //the api key to call lbs service, like tencent map or gaode map
+ LBSServiceKey string `env:"STARHUB_SERVER_LBS_SERVICE_KEY, default=123456"`
+ //the cdn domain for different city
+ CityToCdnDomain map[string]string `env:"STARHUB_SERVER_CITY_TO_CDN_DOMAIN, default="`
+ //the master host
+ IsMasterHost bool `env:"STARHUB_SERVER_IS_MASTER_HOST, default=true"`
+
+ APIServer struct {
+ Port int `env:"STARHUB_SERVER_SERVER_PORT, default=8080"`
+ PublicDomain string `env:"STARHUB_SERVER_PUBLIC_DOMAIN, default=http://localhost:8080"`
+ SSHDomain string `env:"STARHUB_SERVER_SSH_DOMAIN, default=ssh://git@localhost:2222"`
+ }
+
+ Mirror struct {
+ URL string `env:"STARHUB_SERVER_MIRROR_URL, default=http://localhost:8085"`
+ Token string `env:"STARHUB_SERVER_MIRROR_Token, default="`
+ Port int `env:"STARHUB_SERVER_MIRROR_PORT, default=8085"`
+ SessionSecretKey string `env:"STARHUB_SERVER_MIRROR_SESSION_SECRET_KEY, default=mirror"`
+ ProxyURL string `env:"STARHUB_SERVER_MIRROR_PROXY_URL, default="`
+ Remote bool `env:"STARHUB_SERVER_MIRROR_REMOTE, default=true"`
+ WorkerNumber int `env:"STARHUB_SERVER_MIRROR_WORKER_NUMBER, default=5"`
+ PartSize int `env:"STARHUB_SERVER_MIRROR_PART_SIZE, default=100"`
+ LfsConcurrency int `env:"STARHUB_SERVER_MIRROR_LFS_CONCURRENCY, default=5"`
+ // The token number to add to bucket each second
+ RateLimit float64 `env:"STARHUB_SERVER_MIRROR_RATE_LIMIT, default=0.2"`
+ // The capacity of token bucket
+ RateBucketCapacity int `env:"STARHUB_SERVER_MIRROR_RATE_BUCKET_CAPACITY, default=1"`
+ }
+
+ DocsHost string `env:"STARHUB_SERVER_SERVER_DOCS_HOST, default=http://localhost:6636"`
+
+ Database struct {
+ Driver string `env:"STARHUB_DATABASE_DRIVER, default=pg"`
+ DSN string `env:"STARHUB_DATABASE_DSN, default=postgresql://postgres:postgres@localhost:5432/starhub_server?sslmode=disable"`
+ TimeZone string `env:"STARHUB_DATABASE_TIMEZONE, default=Asia/Shanghai"`
+ }
+
+ Redis struct {
+ Endpoint string `env:"STARHUB_SERVER_REDIS_ENDPOINT, default=localhost:6379"`
+ MaxRetries int `env:"STARHUB_SERVER_REDIS_MAX_RETRIES, default=3"`
+ MinIdleConnections int `env:"STARHUB_SERVER_REDIS_MIN_IDLE_CONNECTIONS, default=0"`
+ User string `env:"STARHUB_SERVER_REDIS_USER"`
+ Password string `env:"STARHUB_SERVER_REDIS_PASSWORD"`
+ SentinelMode bool `env:"STARHUB_SERVER_REDIS_USE_SENTINEL, default=false"`
+ SentinelMaster string `env:"STARHUB_SERVER_REDIS_SENTINEL_MASTER"`
+ SentinelEndpoint string `env:"STARHUB_SERVER_REDIS_SENTINEL_ENDPOINT"`
+ }
+
+ GitServer struct {
+ URL string `env:"STARHUB_SERVER_GITSERVER_URL, default=http://localhost:3000"`
+ Type string `env:"STARHUB_SERVER_GITSERVER_TYPE, default=gitea"`
+ Host string `env:"STARHUB_SERVER_GITSERVER_HOST, default=http://localhost:3000"`
+ SecretKey string `env:"STARHUB_SERVER_GITSERVER_SECRET_KEY, default=619c849c49e03754454ccd4cda79a209ce0b30b3"`
+ Username string `env:"STARHUB_SERVER_GITSERVER_USERNAME, default=root"`
+ Password string `env:"STARHUB_SERVER_GITSERVER_PASSWORD, default=password123"`
+ TimeoutSEC int `env:"STARHUB_SERVER_GITSERVER_TIMEOUT_SEC, default=5"`
+ }
+
+ GitalyServer struct {
+ Address string `env:"STARHUB_SERVER_GITALY_SERVER_SOCKET, default=tcp://localhost:9999"`
+ Storge string `env:"STARHUB_SERVER_GITALY_STORGE, default=default"`
+ Token string `env:"STARHUB_SERVER_GITALY_TOKEN, default=abc123secret"`
+ JWTSecret string `env:"STARHUB_SERVER_GITALY_JWT_SECRET, default=signing-key"`
+ }
+
+ MirrorServer struct {
+ Enable bool `env:"STARHUB_SERVER_MIRRORSERVER_ENABLE, default=true"`
+ URL string `env:"STARHUB_SERVER_MIRRORSERVER_URL, default=http://localhost:3000"`
+ Type string `env:"STARHUB_SERVER_MIRRORSERVER_TYPE, default=gitea"`
+ Host string `env:"STARHUB_SERVER_MIRRORSERVER_HOST, default=http://localhost:3000"`
+ SecretKey string `env:"STARHUB_SERVER_MIRRORSERVER_SECRET_KEY, default=619c849c49e03754454ccd4cda79a209ce0b30b3"`
+ Username string `env:"STARHUB_SERVER_MIRRORSERVER_USERNAME, default=root"`
+ Password string `env:"STARHUB_SERVER_MIRRORSERVER_PASSWORD, default=password123"`
+ }
+
+ Frontend struct {
+ URL string `env:"STARHUB_SERVER_FRONTEND_URL, default=https://opencsg.com"`
+ }
+
+ S3 struct {
+ AccessKeyID string `env:"STARHUB_SERVER_S3_ACCESS_KEY_ID, default=H0xkbJj7e86gv2tJYucv"`
+ AccessKeySecret string `env:"STARHUB_SERVER_S3_ACCESS_KEY_SECRET, default=KNmbQNkgxUYwKarIPHeKXAWWgrjTp4fPt0PykXsG"`
+ Region string `env:"STARHUB_SERVER_S3_REGION"`
+ Endpoint string `env:"STARHUB_SERVER_S3_ENDPOINT, default=localhost:9000"`
+ Bucket string `env:"STARHUB_SERVER_S3_BUCKET, default=opencsg-test"`
+ EnableSSL bool `env:"STARHUB_SERVER_S3_ENABLE_SSL, default=false"`
+ }
+
+ SensitiveCheck struct {
+ Enable bool `env:"STARHUB_SERVER_SENSITIVE_CHECK_ENABLE, default=false"`
+ AccessKeyID string `env:"STARHUB_SERVER_SENSITIVE_CHECK_ACCESS_KEY_ID"`
+ AccessKeySecret string `env:"STARHUB_SERVER_SENSITIVE_CHECK_ACCESS_KEY_SECRET"`
+ Region string `env:"STARHUB_SERVER_SENSITIVE_CHECK_REGION"`
+ Endpoint string `env:"STARHUB_SERVER_SENSITIVE_CHECK_ENDPOINT, default=oss-cn-beijing.aliyuncs.com"`
+ EnableSSL bool `env:"STARHUB_SERVER_S3_ENABLE_SSH, default=true"`
+ }
+
+ JWT struct {
+ SigningKey string `env:"STARHUB_JWT_SIGNING_KEY, default=signing-key"`
+ ValidHour int `env:"STARHUB_JWT_VALIDATE_HOUR, default=24"`
+ }
+
+ Space struct {
+ BuilderEndpoint string `env:"STARHUB_SERVER_SPACE_BUILDER_ENDPOINT, default=http://localhost:8081"`
+ // base url for space api running in k8s cluster
+ RunnerEndpoint string `env:"STARHUB_SERVER_SPACE_RUNNER_ENDPOINT, default=http://localhost:8082"`
+ RunnerServerPort int `env:"STARHUB_SERVER_SPACE_RUNNER_SERVER_PORT, default=8082"`
+
+ // the internal root domain will be proxied to, should be internal access only
+ InternalRootDomain string `env:"STARHUB_SERVER_INTERNAL_ROOT_DOMAIN, default=internal.example.com"`
+ // the public root domain will be proxied from
+ PublicRootDomain string `env:"STARHUB_SERVER_PUBLIC_ROOT_DOMAIN, default=public.example.com"`
+ DockerRegBase string `env:"STARHUB_SERVER_DOCKER_REG_BASE, default=registry.cn-beijing.aliyuncs.com/opencsg_public/"`
+ ImagePullSecret string `env:"STARHUB_SERVER_DOCKER_IMAGE_PULL_SECRET, default=opencsg-pull-secret"`
+ // reverse proxy listening port
+ RProxyServerPort int `env:"STARHUB_SERVER_SPACE_RPROXY_SERVER_PORT, default=8083"`
+ // secret key for session encryption
+ SessionSecretKey string `env:"STARHUB_SERVER_SPACE_SESSION_SECRET_KEY, default=secret"`
+ DeployTimeoutInMin int `env:"STARHUB_SERVER_SPACE_DEPLOY_TIMEOUT_IN_MINUTES, default=30"`
+ // gpu model label
+ GPUModelLabel string `env:"STARHUB_SERVER_GPU_MODEL_LABEL, default=aliyun.accelerator/nvidia_name"`
+ ReadnessDelaySeconds int `env:"STARHUB_SERVER_READNESS_DELAY_SECONDS, default=120"`
+ ReadnessPeriodSeconds int `env:"STARHUB_SERVER_READNESS_PERIOD_SECONDS, default=10"`
+ ReadnessFailureThreshold int `env:"STARHUB_SERVER_READNESS_FAILURE_THRESHOLD, default=3"`
+ }
+
+ Model struct {
+ DeployTimeoutInMin int `env:"STARHUB_SERVER_MODEL_DEPLOY_TIMEOUT_IN_MINUTES, default=60"`
+ DownloadEndpoint string `env:"STARHUB_SERVER_MODEL_DOWNLOAD_ENDPOINT, default=https://hub.opencsg.com"`
+ DockerRegBase string `env:"STARHUB_SERVER_MODEL_DOCKER_REG_BASE, default=opencsg-registry.cn-beijing.cr.aliyuncs.com/public/"`
+ NimDockerSecretName string `env:"STARHUB_SERVER_MODEL_NIM_DOCKER_SECRET_NAME, default=ngc-secret"`
+ NimNGCSecretName string `env:"STARHUB_SERVER_MODEL_NIM_NGC_SECRET_NAME, default=nvidia-nim-secrets"`
+ }
+ // send events
+ Event struct {
+ SyncInterval int `env:"STARHUB_SERVER_SYNC_IN_MINUTES, default=1"`
+ }
+
+ Casdoor struct {
+ ClientID string `env:"STARHUB_SERVER_CASDOOR_CLIENT_ID, default=client_id"`
+ ClientSecret string `env:"STARHUB_SERVER_CASDOOR_CLIENT_SECRET, default=client_secret"`
+ Endpoint string `env:"STARHUB_SERVER_CASDOOR_ENDPOINT, default=http://localhost:80"`
+ Certificate string `env:"STARHUB_SERVER_CASDOOR_CERTIFICATE, default=/etc/casdoor/certificate.pem"`
+ OrganizationName string `env:"STARHUB_SERVER_CASDOOR_ORGANIZATION_NAME, default=opencsg"`
+ ApplicationName string `env:"STARHUB_SERVER_CASDOOR_APPLICATION_NAME, default=opencsg"`
+ }
+
+ Nats struct {
+ URL string `env:"OPENCSG_ACCOUNTING_NATS_URL, default=nats://account:g98dc5FA8v4J7ck90w@natsmaster:4222"`
+ MsgFetchTimeoutInSEC int `env:"OPENCSG_ACCOUNTING_MSG_FETCH_TIMEOUTINSEC, default=5"`
+ FeeNotifyNoBalanceSubject string `env:"OPENCSG_ACCOUNTING_NOTIFY_NOBALANCE_SUBJECT, default=accounting.notify.nobalance"`
+ FeeRequestSubject string `env:"OPENCSG_ACCOUNTING_FEE_EVENT_SUBJECT, default=accounting.fee.>"`
+ FeeSendSubject string `env:"STARHUB_SERVER_FEE_SEND_SUBJECT, default=accounting.fee.credit"`
+ TokenSendSubject string `env:"STARHUB_SERVER_TOKEN_SEND_SUBJECT, default=accounting.fee.token"`
+ QuotaSendSubject string `env:"STARHUB_SERVER_QUOTA_SEND_SUBJECT, default=accounting.fee.quota"`
+ MeterRequestSubject string `env:"OPENCSG_ACCOUNTING_METER_EVENT_SUBJECT, default=accounting.metering.>"`
+ MeterDurationSendSubject string `env:"STARHUB_SERVER_METER_DURATION_SEND_SUBJECT, default=accounting.metering.duration"`
+ MeterTokenSendSubject string `env:"STARHUB_SERVER_METER_TOKEN_SEND_SUBJECT, default=accounting.metering.token"`
+ MeterQuotaSendSubject string `env:"STARHUB_SERVER_METER_QUOTA_SEND_SUBJECT, default=accounting.metering.quota"`
+ OrderExpiredSubject string `env:"STARHUB_SERVER_ORDER_EXPIRED_SUBJECT, default=accounting.order.expired"`
+ RechargeSucceedSubject string `env:"STARHUB_SERVER_RECHARGE_SUCCEED_SUBJECT, default=accounting.recharge.succeed"`
+ }
+
+ Accounting struct {
+ Host string `env:"OPENCSG_ACCOUNTING_SERVER_HOST, default=http://localhost"`
+ Port int `env:"OPENCSG_ACCOUNTING_SERVER_PORT, default=8086"`
+ ChargingEnable bool `env:"OPENCSG_ACCOUNTING_CHARGING_ENABLE, default=false"`
+ }
+
+ User struct {
+ Host string `env:"OPENCSG_USER_SERVER_HOST, default=http://localhost"`
+ Port int `env:"OPENCSG_USER_SERVER_PORT, default=8088"`
+ SigninSuccessRedirectURL string `env:"OPENCSG_USER_SERVER_SIGNIN_SUCCESS_REDIRECT_URL, default=http://localhost:3000/server/callback"`
+ CodeSoulerVScodeRedirectURL string `env:"OPENCSG_USER_SERVER_CODESOULER_VSCODE_REDIRECT_URL, default=http://127.0.0.1:37678/callback"`
+ CodeSoulerJetBrainsRedirectURL string `env:"OPENCSG_USER_SERVER_CODESOULER_JETBRAINS_REDIRECT_URL, default=http://127.0.0.1:37679/callback"`
+ }
+
+ MultiSync struct {
+ SaasAPIDomain string `env:"OPENCSG_SAAS_API_DOMAIN, default=https://hub.opencsg.com"`
+ SaasSyncDomain string `env:"OPENCSG_SAAS_SYNC_DOMAIN, default=https://sync.opencsg.com"`
+ DefaultRepoCountLimit int64 `env:"OPENCSG_SAAS_DEFAULT_REPO_COUNT_LIMIT, default=10"`
+ // DefaultRepoSizeLimit and DefaultTrafficLimit is in Bytes
+ DefaultSpeedLimit int64 `env:"OPENCSG_SAAS_DEFAULT_SPEED_LIMIT, default=1024"`
+ DefaultTrafficLimit int64 `env:"OPENCSG_SAAS_DEFAULT_TRAFFIC_LIMIT, default=1024"`
+ Enabled bool `env:"STARHUB_SERVER_MULTI_SYNC_ENABLED, default=false"`
+ }
+
+ Telemetry struct {
+ Enable bool `env:"STARHUB_SERVER_TELEMETRY_ENABLE, default=true"`
+ ReportURL string `env:"STARHUB_SERVER_TELEMETRY_URL, default=http://hub.opencsg.com/api/v1/telemetry"`
+ }
+
+ AutoClean struct {
+ Instance bool `env:"OPENCSG_AUTO_CLEANUP_INSTANCE_ENABLE, default=false"`
+ }
+
+ Dataset struct {
+ PromptMaxJsonlFileSize int64 `env:"OPENCSG_PROMPT_MAX_JSONL_FILESIZE_BYTES, default=1048576"` // 1MB
+ }
+
+ Dataflow struct {
+ Host string `env:"OPENCSG_DATAFLOW_SERVER_HOST, default=http://127.0.0.1"`
+ Port int `env:"OPENCSG_DATAFLOW_SERVER_PORT, default=8000"`
+ }
+
+ Moderation struct {
+ Host string `env:"OPENCSG_MODERATION_SERVER_HOST, default=http://localhost"`
+ Port int `env:"OPENCSG_MODERATION_SERVER_PORT, default=8089"`
+ // comma splitted, and base64 encoded
+ EncodedSensitiveWords string `env:"OPENCSG_MODERATION_SERVER_ENCODED_SENSITIVE_WORDS, default=5Lmg6L+R5bmzLHhpamlucGluZw=="`
+ }
+
+ WorkFLow struct {
+ Endpoint string `env:"OPENCSG_WORKFLOW_SERVER_ENDPOINT, default=localhost:7233"`
+ }
+
+ License struct {
+ PublicKeyFile string `env:"OPENCSG_LICENSE_PUBLIC_KEY_FILE, default=publickey.pem"`
+ PrivateKeyFile string `env:"OPENCSG_LICENSE_PRIVATE_KEY_FILE, default=privatekey.pem"`
+ }
+
+ Argo struct {
+ Namespace string `env:"STARHUB_SERVER_ARGO_NAMESPACE, default=workflows"`
+ // NamespaceQuota is used to create evaluation with free of charge
+ QuotaNamespace string `env:"STARHUB_SERVER_ARGO_QUOTA_NAMESPACE, default=workflows-quota"`
+ QuotaGPUNumber string `env:"STARHUB_SERVER_ARGO_QUOTA_GPU_NUMBER, default=1"`
+ //job will be deleted after JobTTL seconds once the jobs was done
+ JobTTL int `env:"STARHUB_SERVER_ARGO_TTL, default=120"`
+ ServiceAccountName string `env:"STARHUB_SERVER_ARGO_SERVICE_ACCOUNT, default=executor"`
+ // S3PublicBucket is used to store public files, should set bucket same with portal
+ S3PublicBucket string `env:"STARHUB_SERVER_ARGO_S3_PUBLIC_BUCKET"`
+ }
+
+ Payment struct {
+ WXAppId string `env:"STARHUB_SERVER_WXAPPID"`
+ WXMchId string `env:"STARHUB_SERVER_WXMCHID"`
+ WXMchAPIv3Key string `env:"STARHUB_SERVER_WXMCHAPIV3KEY"`
+ WXMchCertificateSerialNumber string `env:"STARHUB_SERVER_WXMCH_CERTIFICATE_SERIAL_NUMBER"`
+ WXMchCertificatePrivateKeyFilePath string `env:"STARHUB_SERVER_WXMCH_CERTIFICATE_PRIVATE_KEY_FILE_PATH"`
+
+ AlipayAppId string `env:"STARHUB_SERVER_ALIPAY_APPID"`
+ AlipayPrivateKey string `env:"STARHUB_SERVER_ALIPAY_PRIVATE_KEY"`
+ AlipayPublicKey string `env:"STARHUB_SERVER_ALIPAY_PUBLIC_KEY"`
+
+ IsProd bool `env:"OPENCSG_PAYMENT_SERVER_IS_PROD, default=false"`
+ PaymentExpireIn int `env:"OPENCSG_PAYMENT_EXPIRE_IN, default=300"`
+ WXPayNotifyPath string `env:"STARHUB_SERVER_WXPAYNOTIFY_PATH, default=/api/v1/payment/wechat/notify"`
+ AlipayNotifyPath string `env:"STARHUB_SERVER_ALIPAYNOTIFY_PATH, default=/api/v1/payment/alipay/notify"`
+ Host string `env:"OPENCSG_PAYMENT_SERVER_HOST, default=http://localhost"`
+ Port int `env:"OPENCSG_PAYMENT_SERVER_PORT, default=8090"`
+ }
+
+ CronJob struct {
+ SyncAsClientCronExpression string `env:"STARHUB_SERVER_CRON_JOB_SYNC_AS_CLIENT_CRON_EXPRESSION, default=0 * * * *"`
+ CalcRecomScoreCronExpression string `env:"STARHUB_SERVER_CRON_JOB_CLAC_RECOM_SCORE_CRON_EXPRESSION, default=0 1 * * *"`
+ }
+}
+
+func SetConfigFile(file string) {
+ configFile = file
+}
+
+func LoadConfig() (*Config, error) {
+ cfg := &Config{}
+
+ if configFile != "" {
+ f, err := os.Open(configFile)
+ if err != nil {
+ return nil, err
+ }
+ err = toml.NewDecoder(f).Decode(cfg)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ // Always read environment variables, even if a config file exists. If a config value is present in both the
+ // config file and the environment, the environment value takes priority. If a config value is missing from
+ // the config file, the default value (specified by the struct field's default tag) will be used.
+ err := envconfig.ProcessWith(context.Background(), &envconfig.Config{
+ Target: cfg,
+ DefaultOverwrite: true,
+ })
+ return cfg, err
+}
+
+
+
package tests
+
+import (
+ "github.com/stretchr/testify/mock"
+ mockdb "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type MockStores struct {
+ User database.UserStore
+ UserLikes database.UserLikesStore
+ Repo database.RepoStore
+ RepoRelation database.RepoRelationsStore
+ Model database.ModelStore
+ SpaceResource database.SpaceResourceStore
+ Tag database.TagStore
+ Dataset database.DatasetStore
+ PromptConversation database.PromptConversationStore
+ PromptPrefix database.PromptPrefixStore
+ LLMConfig database.LLMConfigStore
+ Prompt database.PromptStore
+ Namespace database.NamespaceStore
+ LfsMetaObject database.LfsMetaObjectStore
+ LfsLock database.LfsLockStore
+ Mirror database.MirrorStore
+ MirrorSource database.MirrorSourceStore
+ AccessToken database.AccessTokenStore
+ SyncVersion database.SyncVersionStore
+ SyncClientSetting database.SyncClientSettingStore
+ RuntimeFramework database.RuntimeFrameworksStore
+ DeployTask database.DeployTaskStore
+ UserResources database.UserResourcesStore
+ ClusterInfo database.ClusterInfoStore
+ Code database.CodeStore
+ Collection database.CollectionStore
+ Workflow database.ArgoWorkFlowStore
+ Space database.SpaceStore
+ SpaceSdk database.SpaceSdkStore
+ Recom database.RecomStore
+ RepoRuntimeFramework database.RepositoriesRuntimeFrameworkStore
+ Discussion database.DiscussionStore
+ RuntimeArch database.RuntimeArchitecturesStore
+ ResourceModel database.ResourceModelStore
+ GitServerAccessToken database.GitServerAccessTokenStore
+ Org database.OrgStore
+ MultiSync database.MultiSyncStore
+ File database.FileStore
+ SSH database.SSHKeyStore
+}
+
+func NewMockStores(t interface {
+ Cleanup(func())
+ mock.TestingT
+}) *MockStores {
+ return &MockStores{
+ User: mockdb.NewMockUserStore(t),
+ UserLikes: mockdb.NewMockUserLikesStore(t),
+ Repo: mockdb.NewMockRepoStore(t),
+ RepoRelation: mockdb.NewMockRepoRelationsStore(t),
+ Model: mockdb.NewMockModelStore(t),
+ SpaceResource: mockdb.NewMockSpaceResourceStore(t),
+ Tag: mockdb.NewMockTagStore(t),
+ Dataset: mockdb.NewMockDatasetStore(t),
+ PromptConversation: mockdb.NewMockPromptConversationStore(t),
+ PromptPrefix: mockdb.NewMockPromptPrefixStore(t),
+ LLMConfig: mockdb.NewMockLLMConfigStore(t),
+ Prompt: mockdb.NewMockPromptStore(t),
+ Namespace: mockdb.NewMockNamespaceStore(t),
+ LfsMetaObject: mockdb.NewMockLfsMetaObjectStore(t),
+ LfsLock: mockdb.NewMockLfsLockStore(t),
+ Mirror: mockdb.NewMockMirrorStore(t),
+ MirrorSource: mockdb.NewMockMirrorSourceStore(t),
+ AccessToken: mockdb.NewMockAccessTokenStore(t),
+ SyncVersion: mockdb.NewMockSyncVersionStore(t),
+ SyncClientSetting: mockdb.NewMockSyncClientSettingStore(t),
+ RuntimeFramework: mockdb.NewMockRuntimeFrameworksStore(t),
+ DeployTask: mockdb.NewMockDeployTaskStore(t),
+ UserResources: mockdb.NewMockUserResourcesStore(t),
+ ClusterInfo: mockdb.NewMockClusterInfoStore(t),
+ Code: mockdb.NewMockCodeStore(t),
+ Collection: mockdb.NewMockCollectionStore(t),
+ Workflow: mockdb.NewMockArgoWorkFlowStore(t),
+ Space: mockdb.NewMockSpaceStore(t),
+ SpaceSdk: mockdb.NewMockSpaceSdkStore(t),
+ Recom: mockdb.NewMockRecomStore(t),
+ RepoRuntimeFramework: mockdb.NewMockRepositoriesRuntimeFrameworkStore(t),
+ Discussion: mockdb.NewMockDiscussionStore(t),
+ RuntimeArch: mockdb.NewMockRuntimeArchitecturesStore(t),
+ ResourceModel: mockdb.NewMockResourceModelStore(t),
+ GitServerAccessToken: mockdb.NewMockGitServerAccessTokenStore(t),
+ Org: mockdb.NewMockOrgStore(t),
+ MultiSync: mockdb.NewMockMultiSyncStore(t),
+ File: mockdb.NewMockFileStore(t),
+ SSH: mockdb.NewMockSSHKeyStore(t),
+ }
+}
+
+func (s *MockStores) UserMock() *mockdb.MockUserStore {
+ return s.User.(*mockdb.MockUserStore)
+}
+
+func (s *MockStores) UserLikesMock() *mockdb.MockUserLikesStore {
+ return s.UserLikes.(*mockdb.MockUserLikesStore)
+}
+
+func (s *MockStores) RepoMock() *mockdb.MockRepoStore {
+ return s.Repo.(*mockdb.MockRepoStore)
+}
+
+func (s *MockStores) RepoRelationMock() *mockdb.MockRepoRelationsStore {
+ return s.RepoRelation.(*mockdb.MockRepoRelationsStore)
+}
+
+func (s *MockStores) ModelMock() *mockdb.MockModelStore {
+ return s.Model.(*mockdb.MockModelStore)
+}
+
+func (s *MockStores) SpaceResourceMock() *mockdb.MockSpaceResourceStore {
+ return s.SpaceResource.(*mockdb.MockSpaceResourceStore)
+}
+
+func (s *MockStores) TagMock() *mockdb.MockTagStore {
+ return s.Tag.(*mockdb.MockTagStore)
+}
+
+func (s *MockStores) DatasetMock() *mockdb.MockDatasetStore {
+ return s.Dataset.(*mockdb.MockDatasetStore)
+}
+
+func (s *MockStores) PromptConversationMock() *mockdb.MockPromptConversationStore {
+ return s.PromptConversation.(*mockdb.MockPromptConversationStore)
+}
+
+func (s *MockStores) PromptPrefixMock() *mockdb.MockPromptPrefixStore {
+ return s.PromptPrefix.(*mockdb.MockPromptPrefixStore)
+}
+
+func (s *MockStores) LLMConfigMock() *mockdb.MockLLMConfigStore {
+ return s.LLMConfig.(*mockdb.MockLLMConfigStore)
+}
+
+func (s *MockStores) PromptMock() *mockdb.MockPromptStore {
+ return s.Prompt.(*mockdb.MockPromptStore)
+}
+
+func (s *MockStores) NamespaceMock() *mockdb.MockNamespaceStore {
+ return s.Namespace.(*mockdb.MockNamespaceStore)
+}
+
+func (s *MockStores) LfsMetaObjectMock() *mockdb.MockLfsMetaObjectStore {
+ return s.LfsMetaObject.(*mockdb.MockLfsMetaObjectStore)
+}
+
+func (s *MockStores) LfsLockMock() *mockdb.MockLfsLockStore {
+ return s.LfsLock.(*mockdb.MockLfsLockStore)
+}
+
+func (s *MockStores) MirrorMock() *mockdb.MockMirrorStore {
+ return s.Mirror.(*mockdb.MockMirrorStore)
+}
+
+func (s *MockStores) MirrorSourceMock() *mockdb.MockMirrorSourceStore {
+ return s.MirrorSource.(*mockdb.MockMirrorSourceStore)
+}
+
+func (s *MockStores) AccessTokenMock() *mockdb.MockAccessTokenStore {
+ return s.AccessToken.(*mockdb.MockAccessTokenStore)
+}
+
+func (s *MockStores) SyncVersionMock() *mockdb.MockSyncVersionStore {
+ return s.SyncVersion.(*mockdb.MockSyncVersionStore)
+}
+
+func (s *MockStores) SyncClientSettingMock() *mockdb.MockSyncClientSettingStore {
+ return s.SyncClientSetting.(*mockdb.MockSyncClientSettingStore)
+}
+
+func (s *MockStores) RuntimeFrameworkMock() *mockdb.MockRuntimeFrameworksStore {
+ return s.RuntimeFramework.(*mockdb.MockRuntimeFrameworksStore)
+}
+
+func (s *MockStores) DeployTaskMock() *mockdb.MockDeployTaskStore {
+ return s.DeployTask.(*mockdb.MockDeployTaskStore)
+}
+
+func (s *MockStores) UserResourcesMock() *mockdb.MockUserResourcesStore {
+ return s.UserResources.(*mockdb.MockUserResourcesStore)
+}
+
+func (s *MockStores) ClusterInfoMock() *mockdb.MockClusterInfoStore {
+ return s.ClusterInfo.(*mockdb.MockClusterInfoStore)
+}
+
+func (s *MockStores) CodeMock() *mockdb.MockCodeStore {
+ return s.Code.(*mockdb.MockCodeStore)
+}
+
+func (s *MockStores) CollectionMock() *mockdb.MockCollectionStore {
+ return s.Collection.(*mockdb.MockCollectionStore)
+}
+
+func (s *MockStores) WorkflowMock() *mockdb.MockArgoWorkFlowStore {
+ return s.Workflow.(*mockdb.MockArgoWorkFlowStore)
+}
+
+func (s *MockStores) SpaceMock() *mockdb.MockSpaceStore {
+ return s.Space.(*mockdb.MockSpaceStore)
+}
+
+func (s *MockStores) SpaceSdkMock() *mockdb.MockSpaceSdkStore {
+ return s.SpaceSdk.(*mockdb.MockSpaceSdkStore)
+}
+
+func (s *MockStores) RecomMock() *mockdb.MockRecomStore {
+ return s.Recom.(*mockdb.MockRecomStore)
+}
+
+func (s *MockStores) RepoRuntimeFrameworkMock() *mockdb.MockRepositoriesRuntimeFrameworkStore {
+ return s.RepoRuntimeFramework.(*mockdb.MockRepositoriesRuntimeFrameworkStore)
+}
+
+func (s *MockStores) DiscussionMock() *mockdb.MockDiscussionStore {
+ return s.Discussion.(*mockdb.MockDiscussionStore)
+}
+
+func (s *MockStores) RuntimeArchMock() *mockdb.MockRuntimeArchitecturesStore {
+ return s.RuntimeArch.(*mockdb.MockRuntimeArchitecturesStore)
+}
+
+func (s *MockStores) ResourceModelMock() *mockdb.MockResourceModelStore {
+ return s.ResourceModel.(*mockdb.MockResourceModelStore)
+}
+
+func (s *MockStores) GitServerAccessTokenMock() *mockdb.MockGitServerAccessTokenStore {
+ return s.GitServerAccessToken.(*mockdb.MockGitServerAccessTokenStore)
+}
+
+func (s *MockStores) OrgMock() *mockdb.MockOrgStore {
+ return s.Org.(*mockdb.MockOrgStore)
+}
+
+func (s *MockStores) MultiSyncMock() *mockdb.MockMultiSyncStore {
+ return s.MultiSync.(*mockdb.MockMultiSyncStore)
+}
+
+func (s *MockStores) FileMock() *mockdb.MockFileStore {
+ return s.File.(*mockdb.MockFileStore)
+}
+
+func (s *MockStores) SSHMock() *mockdb.MockSSHKeyStore {
+ return s.SSH.(*mockdb.MockSSHKeyStore)
+}
+
+
+
package tests
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "os"
+ "sync"
+ "time"
+
+ "github.com/DATA-DOG/go-txdb"
+ "github.com/google/uuid"
+ "github.com/testcontainers/testcontainers-go"
+ "github.com/testcontainers/testcontainers-go/modules/postgres"
+ "github.com/testcontainers/testcontainers-go/wait"
+ "github.com/uptrace/bun"
+ "github.com/uptrace/bun/dialect/pgdialect"
+ "github.com/uptrace/bun/driver/pgdriver"
+ "github.com/uptrace/bun/extra/bundebug"
+ "github.com/uptrace/bun/migrate"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/database/migrations"
+)
+
+// This is a modified version of db.go NewDB method, used in test only.
+func newBun(ctx context.Context, config database.DBConfig, useTxdb bool) (bunDB *bun.DB, err error) {
+ switch config.Dialect {
+ case database.DialectPostgres:
+ var sqlDB *sql.DB
+ if useTxdb {
+ sqlDB = sql.OpenDB(txdb.New("pg", config.DSN))
+ } else {
+ sqlDB = sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(config.DSN)))
+ }
+ bunDB = bun.NewDB(sqlDB, pgdialect.New(), bun.WithDiscardUnknownColumns())
+ default:
+ err = fmt.Errorf("unknown database dialect %q", config.Dialect)
+ return
+ }
+
+ err = bunDB.PingContext(ctx)
+ if err != nil {
+ err = fmt.Errorf("pinging %s database: %w", config.Dialect, err)
+ return
+ }
+
+ bunDB.RegisterModel((*database.RepositoryTag)(nil))
+ bunDB.RegisterModel((*database.CollectionRepository)(nil))
+ return
+}
+
+var chMu sync.Mutex
+
+func chProjectRoot() {
+ chMu.Lock()
+ defer chMu.Unlock()
+ for {
+ _, err := os.Stat("builder/store/database/migrations")
+ if err != nil {
+ err = os.Chdir("../")
+ if err != nil {
+ panic(err)
+ }
+ continue
+ }
+ return
+ }
+}
+
+// Init a test db, must call `defer db.Close()` in the test
+func InitTestDB() *database.DB {
+ ctx := context.TODO()
+ // reuse the container, so we don't need to recreate the db for each test
+ // https://github.com/testcontainers/testcontainers-go/issues/2726
+ reuse := testcontainers.CustomizeRequestOption(
+ func(req *testcontainers.GenericContainerRequest) error {
+ req.Reuse = true
+ req.Name = "csghub_test"
+ return nil
+ },
+ )
+
+ pc, err := postgres.Run(ctx,
+ "postgres:15.7",
+ reuse,
+ postgres.WithDatabase("csghub_test"),
+ testcontainers.WithWaitStrategy(
+ wait.ForLog("database system is ready to accept connections").
+ WithOccurrence(2).
+ WithStartupTimeout(5*time.Second)))
+ if err != nil {
+ panic(err)
+ }
+
+ // testcontainers will create a random dsn eachtime
+ dsn, err := pc.ConnectionString(ctx)
+ if err != nil {
+ panic(err)
+ }
+ chProjectRoot()
+ bdb, err := newBun(ctx, database.DBConfig{
+ Dialect: database.DialectPostgres,
+ DSN: dsn + "sslmode=disable",
+ }, false)
+ if err != nil {
+ panic(err)
+ }
+ defer bdb.Close()
+ bdb.AddQueryHook(bundebug.NewQueryHook(
+ bundebug.WithEnabled(false),
+
+ // BUNDEBUG=1 logs failed queries
+ // BUNDEBUG=2 logs all queries
+ bundebug.FromEnv("BUNDEBUG"),
+ ))
+
+ migrator := migrate.NewMigrator(bdb, migrations.Migrations)
+ err = migrator.Init(ctx)
+ if err != nil {
+ panic(err)
+ }
+ _, err = migrator.Migrate(ctx)
+ if err != nil {
+ panic(err)
+ }
+
+ // create a new bun connection with txdb(the `true` param), so all sqls run
+ // using this connection will be wrapped in a Tx automatically.
+ bdb, err = newBun(ctx, database.DBConfig{
+ Dialect: database.DialectPostgres,
+ DSN: dsn + "sslmode=disable",
+ }, true)
+ if err != nil {
+ panic(err)
+ }
+ bdb.AddQueryHook(bundebug.NewQueryHook(
+ bundebug.WithEnabled(false),
+ bundebug.FromEnv("BUNDEBUG"),
+ ))
+
+ return &database.DB{
+ Operator: database.Operator{Core: bdb},
+ BunDB: bdb,
+ }
+}
+
+// Create a random test postgres Database without txdb,
+// this method is *MUCH SLOWER* than TestDB, use it only when you need to testing concurrent
+// transaction execution(e.g., test concurrent select for update locks).
+func InitTransactionTestDB() *database.DB {
+ ctx := context.TODO()
+ cname := "csghub_test_tx_" + uuid.New().String()
+ // reuse the container, so we don't need to recreate the db for each test
+ // https://github.com/testcontainers/testcontainers-go/issues/2726
+ reuse := testcontainers.CustomizeRequestOption(
+ func(req *testcontainers.GenericContainerRequest) error {
+ req.Reuse = true
+ req.Name = cname
+ return nil
+ },
+ )
+
+ pc, err := postgres.Run(ctx, "postgres:15.7", reuse, postgres.WithDatabase(cname),
+ testcontainers.WithWaitStrategy(
+ wait.ForLog("database system is ready to accept connections").
+ WithOccurrence(2).
+ WithStartupTimeout(5*time.Second)))
+ if err != nil {
+ panic(err)
+ }
+
+ // testcontainers will create a random dsn eachtime
+ dsn, err := pc.ConnectionString(ctx)
+ if err != nil {
+ panic(err)
+ }
+
+ chProjectRoot()
+
+ bdb, err := newBun(ctx, database.DBConfig{
+ Dialect: database.DialectPostgres,
+ DSN: dsn + "sslmode=disable",
+ }, false)
+ if err != nil {
+ panic(err)
+ }
+ defer bdb.Close()
+ bdb.AddQueryHook(bundebug.NewQueryHook(
+ bundebug.WithEnabled(false),
+
+ bundebug.FromEnv("BUNDEBUG"),
+ ))
+
+ migrator := migrate.NewMigrator(bdb, migrations.Migrations)
+ err = migrator.Init(ctx)
+ if err != nil {
+ panic(err)
+ }
+ _, err = migrator.Migrate(ctx)
+ if err != nil {
+ panic(err)
+ }
+ bdb, err = newBun(ctx, database.DBConfig{
+ Dialect: database.DialectPostgres,
+ DSN: dsn + "sslmode=disable",
+ }, false)
+ if err != nil {
+ panic(err)
+ }
+
+ return &database.DB{
+ Operator: database.Operator{Core: bdb},
+ BunDB: bdb,
+ }
+}
+
+
+
package types
+
+import (
+ "time"
+
+ "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+)
+
+var WorkFlowFinished = map[v1alpha1.WorkflowPhase]struct{}{
+ v1alpha1.WorkflowSucceeded: {},
+ v1alpha1.WorkflowFailed: {},
+ v1alpha1.WorkflowError: {},
+}
+
+var _ SensitiveRequestV2 = (*EvaluationReq)(nil)
+
+func (c *EvaluationReq) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "task_name",
+ Value: func() string {
+ return c.TaskName
+ },
+ Scenario: "nickname_detection",
+ },
+ {
+ Name: "task_desc",
+ Value: func() string {
+ return c.TaskDesc
+ },
+ Scenario: "comment_detection",
+ },
+ }
+}
+
+type TaskType string
+
+const (
+ TaskTypeEvaluation TaskType = "evaluation"
+ TaskTypeTraining TaskType = "training"
+ TaskTypeComparison TaskType = "comparison"
+ TaskTypeLeaderBoard TaskType = "leaderboard"
+)
+
+type EvaluationReq struct {
+ Username string `json:"-"`
+ TaskName string `json:"task_name"`
+ TaskDesc string `json:"task_desc"`
+ RuntimeFrameworkId int64 `json:"runtime_framework_id"` // ArgoWorkFlow framework
+ Datasets []string `json:"datasets,omitempty"`
+ ResourceId int64 `json:"resource_id"`
+ ModelId string `json:"model_id"`
+ ShareMode bool `json:"share_mode"`
+ Token string `json:"-"`
+ Hardware HardWare `json:"-"`
+ UserUUID string `json:"-"`
+ ClusterID string `json:"-"`
+ Image string `json:"-"`
+ RepoType string `json:"-"`
+ TaskType TaskType `json:"-"`
+ DownloadEndpoint string `json:"-"`
+ ResourceName string `json:"-"`
+}
+
+type ArgoFlowTemplate struct {
+ Name string `json:"name"`
+ Image string `json:"image"`
+ Command []string `json:"command"`
+ Args []string `json:"args"`
+ HardWare HardWare `json:"hardware,omitempty"`
+ Env map[string]string `json:"env,omitempty"`
+ Annotation map[string]string `json:"annotation,omitempty"`
+}
+
+type ArgoWorkFlowReq struct {
+ ClusterID string `json:"cluster_id"`
+ RepoType string `json:"repo_type"`
+ Templates []ArgoFlowTemplate `json:"templates,omitempty"`
+ Entrypoint string `json:"entrypoint"`
+ Username string `json:"username"`
+ TaskName string `json:"task_name"`
+ TaskId string `json:"task_id"`
+ TaskType TaskType `json:"task_type"`
+ RepoIds []string `json:"repo_ids"`
+ TaskDesc string `json:"task_desc"`
+ Image string `json:"image"` // ArgoWorkFlow framework
+ Datasets []string `json:"datasets,omitempty"`
+ ResourceId int64 `json:"resource_id"`
+ ResourceName string `json:"resource_name"`
+ UserUUID string `json:"user_uuid"`
+ ShareMode bool `json:"share_mode"`
+}
+
+type ArgoWorkFlowListRes struct {
+ List []ArgoWorkFlowRes `json:"list"`
+ Total int `json:"total"`
+}
+
+type ArgoWorkFlowRes struct {
+ ID int64 `json:"id"`
+ RepoIds []string `json:"repo_ids"`
+ RepoType string `json:"repo_type,omitempty"`
+ Username string `json:"username"`
+ TaskName string `json:"task_name"`
+ TaskId string `json:"task_id"`
+ TaskType TaskType `json:"task_type"`
+ TaskDesc string `json:"task_desc"`
+ Datasets []string `json:"datasets,omitempty"`
+ ResourceId int64 `json:"resource_id,omitempty"`
+ Status v1alpha1.WorkflowPhase `json:"status"`
+ Reason string `json:"reason,omitempty"`
+ Image string `bun:",notnull" json:"image"`
+ SubmitTime time.Time `json:"submit_time"`
+ StartTime time.Time `json:"start_time,omitempty"`
+ EndTime time.Time `json:"end_time,omitempty"`
+ ResultURL string `json:"result_url"`
+ DownloadURL string `json:"download_url"`
+ FailuresURL string `json:"failures_url"`
+}
+
+type RepoTags struct {
+ RepoId string `json:"repo_id"`
+ Tags []RepoTag `json:"tags"`
+}
+
+type EvaluationRes struct {
+ ID int64 `json:"id"`
+ RepoIds []string `json:"repo_ids"`
+ RepoType string `json:"repo_type,omitempty"`
+ Username string `json:"username"`
+ TaskName string `json:"task_name"`
+ TaskId string `json:"task_id"`
+ TaskType TaskType `json:"task_type"`
+ TaskDesc string `json:"task_desc"`
+ Datasets []RepoTags `json:"datasets,omitempty"`
+ ResourceId int64 `json:"resource_id,omitempty"`
+ Status string `json:"status"`
+ Reason string `json:"reason,omitempty"`
+ Image string `bun:",notnull" json:"image"`
+ SubmitTime time.Time `json:"submit_time"`
+ StartTime time.Time `json:"start_time,omitempty"`
+ EndTime time.Time `json:"end_time,omitempty"`
+ ResultURL string `json:"result_url"`
+ DownloadURL string `json:"download_url"`
+ FailuresURL string `json:"failures_url"`
+}
+
+type (
+ EvaluationDelReq = ArgoWorkFlowDeleteReq
+ EvaluationGetReq = ArgoWorkFlowDeleteReq
+ ArgoWorkFlowGetReq = ArgoWorkFlowDeleteReq
+)
+
+type ArgoWorkFlowDeleteReq struct {
+ ID int64 `json:"id"`
+ Username string `json:"username"`
+}
+
+
+
package types
+
+import (
+ "strings"
+ "time"
+)
+
+var CollectionSorts = []string{"trending", "recently_update", "most_favorite"}
+
+type Collection struct {
+ ID int64 `json:"id"`
+ Username string `json:"username"`
+ Namespace string `json:"namespace"`
+ Theme string `json:"theme"`
+ Name string `json:"name"`
+ Nickname string `json:"nickname"`
+ Description string `json:"description"`
+ Private bool `json:"private"`
+ Repositories []CollectionRepository `json:"repositories"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Likes int64 `json:"likes"`
+ UserLikes bool `json:"user_likes"`
+ CanWrite bool `json:"can_write"`
+ CanManage bool `json:"can_manage"`
+ Avatar string `json:"avatar"`
+}
+
+type CollectionRepository struct {
+ ID int64 `json:"id"`
+ UserID int64 `json:"user_id"`
+ Path string `json:"path"`
+ Name string `json:"name"`
+ Nickname string `json:"nickname"`
+ Description string `json:"description"`
+ Private bool `json:"private"`
+ Likes int64 `json:"likes"`
+ DownloadCount int64 `json:"download_count"`
+ Tags []RepoTag `json:"tags"`
+ RepositoryType RepositoryType `json:"repository_type"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Status string `json:"status,omitempty"`
+}
+
+type UpdateCollectionReposReq struct {
+ RepoIDs []int64 `json:"repo_ids"`
+ Username string `json:"-"`
+ UserID int64 `json:"-"`
+ ID int64 `json:"-"` //collection ID
+}
+
+type CreateCollectionReq struct {
+ Username string `json:"-"`
+ UserUUID string `json:"-"`
+ UserID int64 `json:"-"`
+ ID int64 `json:"-"`
+ Namespace string `json:"namespace" example:"user_or_org_name"`
+ Theme string `json:"theme" example:"#fff000"`
+ Name string `json:"name" example:"collection1"`
+ Nickname string `json:"nickname" example:"collection nick name"`
+ Description string `json:"description"`
+ Private bool `json:"private"`
+}
+
+// CreateCollectionReq implements SensitiveRequestV2
+func (c *CreateCollectionReq) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "name",
+ Value: func() string {
+ return c.Name
+ },
+ Scenario: "nickname_detection",
+ },
+ {
+ Name: "nickname",
+ Value: func() string {
+ return c.Nickname
+ },
+ Scenario: "nickname_detection",
+ },
+ {
+ Name: "description",
+ Value: func() string {
+ return c.Description
+ },
+ Scenario: "comment_detection",
+ },
+ }
+}
+
+// NamespaceAndName returns namespace and name by parsing repository path
+func (r CollectionRepository) NamespaceAndName() (namespace string, name string) {
+ fields := strings.Split(r.Path, "/")
+ return fields[0], fields[1]
+}
+
+type CollectionFilter struct {
+ Sort string
+ Search string
+}
+
+
+
package types
+
+import (
+ "errors"
+ "net/http"
+ "path"
+ "regexp"
+ "time"
+
+ pb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+)
+
+const LfsMediaType = "application/vnd.git-lfs+json"
+
+var (
+ oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
+ ErrHashMismatch = errors.New("content hash does not match OID")
+ ErrSizeMismatch = errors.New("content size does not match")
+)
+
+type InfoRefsReq struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+ Rpc string `json:"rpc"`
+ GitProtocol string `json:"git_protocol"`
+ CurrentUser string `json:"current_user"`
+}
+
+type GitUploadPackReq struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+ GitProtocol string `json:"git_protocol"`
+ Request *http.Request `json:"request"`
+ Writer http.ResponseWriter `json:"writer"`
+ CurrentUser string `json:"current_user"`
+}
+
+type GitReceivePackReq = GitUploadPackReq
+
+type BatchRequest struct {
+ Operation string `json:"operation"`
+ Transfers []string `json:"transfers,omitempty"`
+ Ref *Reference `json:"ref,omitempty"`
+ Objects []Pointer `json:"objects"`
+ Authorization string `json:"authorization"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+ CurrentUser string `json:"current_user"`
+}
+
+type UploadRequest struct {
+ Oid string `json:"oid"`
+ Size int64 `json:"size"`
+ CurrentUser string `json:"current_user"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+}
+
+type DownloadRequest struct {
+ Oid string `json:"oid"`
+ Size int64 `json:"size"`
+ CurrentUser string `json:"current_user"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+ SaveAs string `json:"save_as"`
+}
+
+type VerifyRequest struct {
+ CurrentUser string `json:"current_user"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+}
+
+type Reference struct {
+ Name string `json:"name"`
+}
+
+type Pointer struct {
+ Oid string `json:"oid"`
+ Size int64 `json:"size"`
+ DownloadURL string `json:"download_url"`
+}
+
+type BatchResponse struct {
+ Transfer string `json:"transfer,omitempty"`
+ Objects []*ObjectResponse `json:"objects"`
+}
+
+type Link struct {
+ Href string `json:"href"`
+ Header map[string]string `json:"header,omitempty"`
+ ExpiresAt *time.Time `json:"expires_at,omitempty"`
+}
+
+type ObjectResponse struct {
+ Pointer
+ Actions map[string]*Link `json:"actions,omitempty"`
+ Error *ObjectError `json:"error,omitempty"`
+}
+
+type ObjectError struct {
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+type SSHAllowedReq struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+ Action string `json:"action"`
+ Repo string `json:"project"`
+ Changes string `json:"changes"`
+ Protocol string `json:"protocol"`
+ KeyID string `json:"key_id,omitempty"`
+ Username string `json:"username,omitempty"`
+ Krb5Principal string `json:"krb5principal,omitempty"`
+ CheckIP string `json:"check_ip,omitempty"`
+ NamespacePath string `json:"namespace_path,omitempty"`
+}
+
+type SSHAllowedResp struct {
+ Success bool `json:"status"`
+ Message string `json:"message"`
+ Repo string `json:"gl_repository"`
+ UserID string `json:"gl_id"`
+ KeyType string `json:"gl_key_type"`
+ KeyID int `json:"gl_key_id"`
+ ProjectID int `json:"gl_project_id"`
+ RootNamespaceID int `json:"gl_root_namespace_id"`
+ Username string `json:"gl_username"`
+ GitConfigOptions []string `json:"git_config_options"`
+ Gitaly Gitaly `json:"gitaly"`
+ GitProtocol string `json:"git_protocol"`
+ Payload CustomPayload `json:"payload"`
+ ConsoleMessages []string `json:"gl_console_messages"`
+ Who string
+ StatusCode int
+ // NeedAudit indicates whether git event should be audited to rails.
+ NeedAudit bool `json:"need_audit"`
+}
+
+type CustomPayload struct {
+ Action string `json:"action"`
+ Data CustomPayloadData `json:"data"`
+}
+
+type CustomPayloadData struct {
+ APIEndpoints []string `json:"api_endpoints"`
+ Username string `json:"gl_username"`
+ PrimaryRepo string `json:"primary_repo"`
+ UserID string `json:"gl_id,omitempty"`
+ RequestHeaders map[string]string `json:"request_headers"`
+ GeoProxyDirectToPrimary bool `json:"geo_proxy_direct_to_primary"`
+ GeoProxyFetchDirectToPrimary bool `json:"geo_proxy_fetch_direct_to_primary"`
+ GeoProxyFetchDirectToPrimaryWithOptions bool `json:"geo_proxy_fetch_direct_to_primary_with_options"`
+ GeoProxyFetchSSHDirectToPrimary bool `json:"geo_proxy_fetch_ssh_direct_to_primary"`
+ GeoProxyPushSSHDirectToPrimary bool `json:"geo_proxy_push_ssh_direct_to_primary"`
+}
+
+type Gitaly struct {
+ Repo pb.Repository `json:"repository"`
+ Address string `json:"address"`
+ Token string `json:"token"`
+ Features map[string]string `json:"features"`
+}
+
+type LfsAuthenticateReq struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType RepositoryType `json:"repo_type"`
+ Operation string `json:"operation"`
+ Repo string `json:"project"`
+ KeyID string `json:"key_id,omitempty"`
+ UserID string `json:"user_id,omitempty"`
+}
+
+type LfsAuthenticateResp struct {
+ Username string `json:"username"`
+ LfsToken string `json:"lfs_token"`
+ RepoPath string `json:"repository_http_path"`
+ ExpiresIn int `json:"expires_in"`
+}
+
+type GitalyAllowedReq struct {
+ Action string `json:"action"`
+ GlRepository string `json:"gl_repository"`
+ Project string `json:"project"`
+ Changes string `json:"changes"`
+ Protocol string `json:"protocol"`
+ Env string `json:"env"`
+ UserID string `json:"user_id"`
+ KeyID string `json:"key_id"`
+ CheckIP string `json:"check_ip"`
+}
+
+func (p Pointer) Valid() bool {
+ if len(p.Oid) != 64 {
+ return false
+ }
+ if !oidPattern.MatchString(p.Oid) {
+ return false
+ }
+ if p.Size < 0 {
+ return false
+ }
+ return true
+}
+
+func (p Pointer) RelativePath() string {
+ if len(p.Oid) < 5 {
+ return p.Oid
+ }
+
+ return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
+}
+
+
+
package types
+
+import (
+ "time"
+)
+
+type RepoBranchCommit struct {
+ ID string `json:"id"`
+}
+
+type CreateModelReq struct {
+ BaseModel string `json:"base_model"`
+ CreateRepoReq
+}
+
+type UpdateModelReq struct {
+ BaseModel *string `json:"base_model"`
+ UpdateRepoReq
+}
+
+type UpdateRepoReq struct {
+ Username string `json:"-"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ RepoType RepositoryType `json:"-"`
+ Nickname *string `json:"nickname" example:"model display name"`
+ Description *string `json:"description"`
+ Private *bool `json:"private" example:"false"`
+ // a sign to indicate if the request is from admin
+ Admin string `json:"-"`
+}
+
+// make sure UpdateModelReq implements SensitiveRequest interface
+var _ SensitiveRequestV2 = (*UpdateRepoReq)(nil)
+
+func (c *UpdateRepoReq) GetSensitiveFields() []SensitiveField {
+ var fields []SensitiveField
+ if c.Nickname != nil {
+ fields = append(fields, SensitiveField{
+ Name: "nickname",
+ Value: func() string {
+ return *c.Nickname
+ },
+ Scenario: "nickname_detection",
+ })
+ }
+ if c.Description != nil {
+ fields = append(fields, SensitiveField{
+ Name: "description",
+ Value: func() string {
+ return *c.Description
+ },
+ Scenario: "comment_detection",
+ })
+ }
+ return fields
+}
+
+type UpdateDownloadsReq struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ ReqDate string `json:"date"`
+ Date time.Time
+ CloneCount int64 `json:"download_count"`
+ RepoType RepositoryType
+}
+
+type ModelPredictReq struct {
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ Input string `json:"input"`
+ Version string `json:"version"`
+ CurrentUser string `json:"current_user"`
+}
+
+type ModelPredictResp struct {
+ Content string `json:"content"`
+ // TODO:add metrics like tokens, latency etc
+}
+
+type CreateRepoReq struct {
+ Username string `json:"-" example:"creator_user_name"`
+ Namespace string `json:"namespace" example:"user_or_org_name"`
+ Name string `json:"name" example:"model_name_1"`
+ Nickname string `json:"nickname" example:"model display name"`
+ Description string `json:"description"`
+ Private bool `json:"private"`
+ Labels string `json:"labels" example:""`
+ License string `json:"license" example:"MIT"`
+ Readme string `json:"readme"`
+ DefaultBranch string `json:"default_branch" example:"main"`
+ RepoType RepositoryType `json:"-"`
+ Admin string `json:"admin"`
+}
+
+// make sure CreateRepoReq implements SensitiveRequest
+var _ SensitiveRequestV2 = (*CreateRepoReq)(nil)
+
+func (c *CreateRepoReq) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "description",
+ Value: func() string {
+ return c.Description
+ },
+ Scenario: "comment_detection",
+ },
+ {
+ Name: "name",
+ Value: func() string {
+ return c.Name
+ },
+ Scenario: "nickname_detection",
+ },
+ {
+ Name: "nickname",
+ Value: func() string {
+ return c.Nickname
+ },
+ Scenario: "nickname_detection",
+ },
+ }
+}
+
+type DeleteRepoReq struct {
+ Username string `json:"username" example:"creator_user_name"`
+ Namespace string `json:"namespace" example:"user_or_org_name"`
+ Name string `json:"name" example:"model_name_1"`
+ RepoType RepositoryType `json:"-"`
+}
+
+type Relations struct {
+ Models []*Model `json:"models,omitempty"`
+ Datasets []*Dataset `json:"datasets,omitempty"`
+ Codes []*Code `json:"codes,omitempty"`
+ Spaces []*Space `json:"spaces,omitempty"`
+ Prompts []*PromptRes `json:"prompts,omitempty"`
+}
+
+type Model struct {
+ ID int64 `json:"id,omitempty"`
+ Name string `json:"name"`
+ Nickname string `json:"nickname"`
+ Description string `json:"description"`
+ Likes int64 `json:"likes"`
+ Downloads int64 `json:"downloads"`
+ Path string `json:"path"`
+ RepositoryID int64 `json:"repository_id"`
+ Private bool `json:"private"`
+ User *User `json:"user,omitempty"`
+ Tags []RepoTag `json:"tags,omitempty"`
+ Readme string `json:"readme"`
+ Repository Repository `json:"repository"`
+ DefaultBranch string `json:"default_branch"`
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ // widget UI style: generation,chat
+ WidgetType ModelWidgetType `json:"widget_type" example:"generation"`
+ // url to interact with the model
+ Status string `json:"status" example:"RUNNING"`
+ UserLikes bool `json:"user_likes"`
+ Source RepositorySource `json:"source"`
+ SyncStatus RepositorySyncStatus `json:"sync_status"`
+ EnableInference bool `json:"enable_inference"`
+ EnableFinetune bool `json:"enable_finetune"`
+ EnableEvaluation bool `json:"enable_evaluation"`
+ BaseModel string `json:"base_model"`
+ License string `json:"license"`
+ CanWrite bool `json:"can_write"`
+ CanManage bool `json:"can_manage"`
+ Namespace *Namespace `json:"namespace"`
+ RecomOpWeight int `json:"recom_op_weight,omitempty"`
+ SensitiveCheckStatus string `json:"sensitive_check_status"`
+ MirrorLastUpdatedAt time.Time `json:"mirror_last_updated_at"`
+}
+
+type SDKModelInfo struct {
+ ID string `json:"id"`
+ Author string `json:"author,omitempty"`
+ // last commit sha
+ Sha string `json:"sha,omitempty"`
+ CreatedAt time.Time `json:"created_at,omitempty"`
+ LastModified time.Time `json:"last_modified,omitempty"`
+ Private bool `json:"private"`
+ Disabled bool `json:"disabled,omitempty"`
+ Gated interface{} `json:"gated,omitempty"` // "auto", "manual", or false
+ Downloads int `json:"downloads"`
+ Likes int `json:"likes"`
+ LibraryName string `json:"library_name,omitempty"`
+ Tags []string `json:"tags"`
+ PipelineTag string `json:"pipeline_tag,omitempty"`
+ MaskToken string `json:"mask_token,omitempty"`
+ WidgetData interface{} `json:"widget_data,omitempty"` // Type Any
+ ModelIndex map[string]interface{} `json:"model_index,omitempty"` // Dict
+ Config map[string]interface{} `json:"config,omitempty"` // Dict
+ TransformersInfo interface{} `json:"transformers_info,omitempty"` // TransformersInfo
+ CardData interface{} `json:"card_data,omitempty"` // ModelCardData
+ Siblings []SDKFile `json:"siblings"`
+ Spaces []string `json:"spaces,omitempty"`
+ SafeTensors interface{} `json:"safetensors,omitempty"` // SafeTensorsInfo
+}
+
+type ModelWidgetType string
+
+const (
+ ModelWidgetTypeGeneration ModelWidgetType = "generation"
+ ModelWidgetTypeChat ModelWidgetType = "chat"
+)
+
+type ModelRunReq struct {
+ DeployName string `json:"deploy_name"`
+ ClusterID string `json:"cluster_id"`
+ Env string `json:"env"`
+ ResourceID int64 `json:"resource_id"`
+ RuntimeFrameworkID int64 `json:"runtime_framework_id"`
+ MinReplica int `json:"min_replica"`
+ MaxReplica int `json:"max_replica"`
+ Revision string `json:"revision"`
+ SecureLevel int `json:"secure_level"`
+ OrderDetailID int64 `json:"order_detail_id"`
+}
+
+var _ SensitiveRequestV2 = (*ModelRunReq)(nil)
+
+func (c *ModelRunReq) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "deploy_name",
+ Value: func() string {
+ return c.DeployName
+ },
+ Scenario: "nickname_detection",
+ },
+ }
+}
+
+type InstanceRunReq struct {
+ DeployName string `json:"deploy_name"`
+ ClusterID string `json:"cluster_id"`
+ ResourceID int64 `json:"resource_id"`
+ RuntimeFrameworkID int64 `json:"runtime_framework_id"`
+ Revision string `json:"revision"`
+ OrderDetailID int64 `json:"order_detail_id"`
+}
+
+var _ SensitiveRequestV2 = (*InstanceRunReq)(nil)
+
+func (c *InstanceRunReq) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "deploy_name",
+ Value: func() string {
+ return c.DeployName
+ },
+ Scenario: "nickname_detection",
+ },
+ }
+}
+
+type ModelUpdateRequest struct {
+ MinReplica int `json:"min_replica"` // min replica of instance/pod
+ MaxReplica int `json:"max_replica"` // max replica of instance/pod
+ Hardware HardWare `json:"hardware"` // resource requirements
+ ImageID string `json:"image_id" binding:"required"`
+ Env map[string]string `json:"env"` // runtime env variables
+ ClusterID string `json:"cluster_id"`
+ SvcName string `json:"svc_name"`
+}
+
+type ModelUpdateResponse struct {
+ DeployID int64 `json:"deploy_id"`
+ Code int `json:"code"`
+ Message string `json:"message"`
+}
+
+type ModelStatusEventData struct {
+ Status string `json:"status"`
+ Details []Instance `json:"details"`
+}
+
+const (
+ SpaceType = iota // space
+ InferenceType = 1 // inference endpoint
+ FinetuneType = 2 // finetune
+ ServerlessType = 3 // serverless
+ EvaluationType = 4 // evaluation
+ UnknownType = -1 // unknown case
+)
+
+type DeployActReq struct {
+ RepoType RepositoryType `json:"repo_type"`
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ CurrentUser string `json:"current_user"`
+ DeployID int64 `json:"deploy_id"`
+ DeployType int `json:"deploy_type"`
+ InstanceName string `json:"instance_name"`
+}
+
+type DeployUpdateReq struct {
+ DeployName *string `json:"deploy_name"`
+ ClusterID *string `json:"cluster_id"`
+ Env *string `json:"env"`
+ ResourceID *int64 `json:"resource_id"`
+ RuntimeFrameworkID *int64 `json:"runtime_framework_id"`
+ MinReplica *int `json:"min_replica" validate:"min=1"`
+ MaxReplica *int `json:"max_replica" validate:"min=1,gtefield=MinReplica"`
+ Revision *string `json:"revision"`
+ SecureLevel *int `json:"secure_level"`
+}
+
+type RelationModels struct {
+ Models []string `json:"models"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+type RelationDatasets struct {
+ Datasets []string `json:"datasets"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+type RelationModel struct {
+ Model string `json:"model"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+type RelationDataset struct {
+ Dataset string `json:"dataset"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+
+
package types
+
+type CreateOrgReq struct {
+ // Org unique identifier
+ Name string `json:"name" example:"org_name_1"`
+ // Display name
+ Nickname string `json:"nickname" example:"org_display_name"`
+ Description string `json:"description" example:"org description"`
+ Username string `json:"-"`
+ Homepage string `json:"homepage,omitempty" example:"https://www.example.com"`
+ Logo string `json:"logo,omitempty" example:"https://www.example.com/logo.png"`
+ Verified bool `json:"verified" example:"false"`
+ OrgType string `json:"org_type" example:"company or school etc"`
+}
+
+// CreateOrgReq implements SensitiveRequestV2
+var _ SensitiveRequestV2 = (*CreateOrgReq)(nil)
+
+func (c *CreateOrgReq) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "name",
+ Value: func() string { return c.Name },
+ Scenario: "nickname_detection",
+ },
+ {
+ Name: "nickname",
+ Value: func() string { return c.Nickname },
+ Scenario: "nickname_detection",
+ },
+ {
+ Name: "description",
+ Value: func() string { return c.Description },
+ Scenario: "comment_detection",
+ },
+ {
+ Name: "homepage",
+ Value: func() string { return c.Homepage },
+ Scenario: "chat_detection",
+ },
+ }
+}
+
+type EditOrgReq struct {
+ // Display name
+ Nickname *string `json:"nickname" example:"org display name"`
+ Description *string `json:"description" example:"org description"`
+ // TODO:rename json field name to 'name", need to negotiate with Portal engineer
+ // Org unique identifier
+ Name string `json:"-"`
+ Homepage *string `json:"homepage,omitempty" example:"https://www.example.com"`
+ Logo *string `json:"logo,omitempty" example:"https://www.example.com/logo.png"`
+ Verified *bool `json:"verified" example:"false"`
+ OrgType *string `json:"org_type" example:"company or school etc"`
+ CurrentUser string `json:"-"`
+}
+
+// EditOrgReq implements SensitiveRequestV2
+var _ SensitiveRequestV2 = (*EditOrgReq)(nil)
+
+func (e *EditOrgReq) GetSensitiveFields() []SensitiveField {
+ var fields []SensitiveField
+ if e.Nickname != nil {
+ fields = append(fields, SensitiveField{
+ Name: "nickname",
+ Value: func() string {
+ return *e.Nickname
+ },
+ Scenario: "nickname_detection",
+ })
+ }
+ if e.Description != nil {
+ fields = append(fields, SensitiveField{
+ Name: "description",
+ Value: func() string {
+ return *e.Description
+ },
+ Scenario: "comment_detection",
+ })
+ }
+ if e.Homepage != nil {
+ fields = append(fields, SensitiveField{
+ Name: "homepage",
+ Value: func() string {
+ return *e.Homepage
+ },
+ Scenario: "chat_detection",
+ })
+ }
+ return fields
+}
+
+type DeleteOrgReq struct {
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+type OrgDatasetsReq struct {
+ // org name of dataset
+ Namespace string `json:"namespace"`
+ CurrentUser string `json:"-"`
+ PageOpts
+}
+
+type (
+ OrgModelsReq = OrgDatasetsReq
+ OrgCodesReq = OrgDatasetsReq
+ OrgSpacesReq = OrgDatasetsReq
+ OrgCollectionsReq = OrgDatasetsReq
+ OrgPromptsReq = OrgDatasetsReq
+)
+
+type Organization struct {
+ // unique name of the organization
+ Name string `json:"path"`
+ Nickname string `json:"name,omitempty"`
+ Homepage string `json:"homepage,omitempty"`
+ Logo string `json:"logo,omitempty"`
+ OrgType string `json:"org_type,omitempty"`
+ Verified bool `json:"verified"`
+ UserID int64 `json:"user_id,omitempty"`
+}
+
+type Member struct {
+ Username string `json:"username"`
+ Nickname string `json:"nickname"`
+ UUID string `json:"uuid"`
+ Avatar string `json:"avatar,omitempty"`
+ Role string `json:"role,omitempty"`
+ LastLoginAt string `json:"last_login_at,omitempty"`
+}
+
+
+
package types
+
+import (
+ "time"
+)
+
+type RepositoryType string
+type RepositorySource string
+type RepositorySyncStatus string
+
+type SensitiveCheckStatus int
+
+// String returns a string representation of the sensitive check status.
+//
+// It returns one of "Fail", "Pending", "Pass", "Skip", "Exception", or "Unknown".
+func (s SensitiveCheckStatus) String() string {
+ switch s {
+ case SensitiveCheckFail:
+ return "Fail"
+ case SensitiveCheckPending:
+ return "Pending"
+ case SensitiveCheckPass:
+ return "Pass"
+ case SensitiveCheckSkip:
+ return "Skip"
+ case SensitiveCheckException:
+ return "Exception"
+ default:
+ return "Unknown"
+ }
+}
+
+const (
+ ResTypeKey string = "hub-res-type"
+ ResNameKey string = "hub-res-name"
+ ResDeployID string = "hub-deploy-id"
+
+ ModelRepo RepositoryType = "model"
+ DatasetRepo RepositoryType = "dataset"
+ SpaceRepo RepositoryType = "space"
+ CodeRepo RepositoryType = "code"
+ PromptRepo RepositoryType = "prompt"
+ UnknownRepo RepositoryType = ""
+
+ OpenCSGSource RepositorySource = "opencsg"
+ LocalSource RepositorySource = "local"
+ HuggingfaceSource RepositorySource = "huggingface"
+
+ SyncStatusPending RepositorySyncStatus = "pending"
+ SyncStatusInProgress RepositorySyncStatus = "inprogress"
+ SyncStatusFailed RepositorySyncStatus = "failed"
+ SyncStatusCompleted RepositorySyncStatus = "completed"
+
+ SensitiveCheckFail SensitiveCheckStatus = -1 //sensitive content detected
+ SensitiveCheckPending SensitiveCheckStatus = 0 //default
+ SensitiveCheckPass SensitiveCheckStatus = 1 //pass
+ SensitiveCheckSkip SensitiveCheckStatus = 2 //skip
+ SensitiveCheckException SensitiveCheckStatus = 3 //error happen
+
+ EndpointPublic int = 1 // public - anyone can access
+ EndpointPrivate int = 2 // private - access with read permission
+
+ MainBranch string = "main"
+
+ ReadmeFileName = "README.md"
+ GitAttributesFileName = ".gitattributes"
+)
+
+type RepoRequest struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ Ref string `json:"ref"`
+ Path string `json:"path"`
+ Page int `json:"page"`
+ Per int `json:"per"`
+}
+
+type Branch struct {
+ Name string `json:"name"`
+ Message string `json:"message"`
+ Commit RepoBranchCommit `json:"commit"`
+}
+
+type Tag struct {
+ Name string `json:"name"`
+ Message string `json:"message"`
+ Commit DatasetTagCommit `json:"commit"`
+}
+
+type Repository struct {
+ HTTPCloneURL string `json:"http_clone_url"`
+ SSHCloneURL string `json:"ssh_clone_url"`
+}
+
+type RepoPageOpts struct {
+ PageOpts
+ PageCount int `json:"page_count"`
+ Total int `json:"total"`
+}
+
+type Instance struct {
+ Name string `json:"name"`
+ Status string `json:"status"`
+}
+
+// repo object(cover model/space/code/dataset) for deployer
+type DeployRepo struct {
+ DeployID int64 `json:"deploy_id,omitempty"`
+ DeployName string `json:"deploy_name,omitempty"`
+ SpaceID int64 `json:"space_id,omitempty"`
+ Path string `json:"model_id,omitempty"` // csghub ask for model_id = namespace/name
+ Namespace string `json:"namespace,omitempty"`
+ Name string `json:"name,omitempty"`
+ Status string `json:"status"`
+ GitPath string `json:"git_path,omitempty"`
+ GitBranch string `json:"git_branch,omitempty"`
+ Sdk string `json:"sdk,omitempty"`
+ SdkVersion string `json:"sdk_version,omitempty"`
+ Env string `json:"env,omitempty"`
+ Secret string `json:"secret,omitempty"`
+ Template string `json:"template,omitempty"`
+ Hardware string `json:"hardware,omitempty"`
+ ImageID string `json:"image_id,omitempty"`
+ UserID int64 `json:"user_id,omitempty"`
+ ModelID int64 `json:"repo_model_id,omitempty"` // for URM code logic
+ RepoID int64 `json:"repository_id,omitempty"`
+ RuntimeFramework string `json:"runtime_framework,omitempty"`
+ ContainerPort int `json:"container_port,omitempty"`
+ Annotation string `json:"annotation,omitempty"`
+ MinReplica int `json:"min_replica,omitempty"`
+ MaxReplica int `json:"max_replica,omitempty"`
+ SvcName string `json:"svc_name,omitempty"`
+ Endpoint string `json:"endpoint,omitempty"`
+ CreatedAt time.Time `json:"created_at,omitempty"`
+ UpdatedAt time.Time `json:"updated_at,omitempty"`
+ ClusterID string `json:"cluster_id,omitempty"`
+ SecureLevel int `json:"secure_level,omitempty"`
+ ActualReplica int `json:"actual_replica,omitempty"`
+ DesiredReplica int `json:"desired_replica,omitempty"`
+ Instances []Instance `json:"instances,omitempty"`
+ InstanceName string `json:"instance_name,omitempty"`
+ Private bool `json:"private"`
+ Type int `json:"type,omitempty"`
+ ProxyEndpoint string `json:"proxy_endpoint,omitempty"`
+ UserUUID string `json:"user_uuid,omitempty"`
+ SKU string `json:"sku,omitempty"`
+ OrderDetailID int64 `json:"order_detail_id,omitempty"`
+ PayMode PayMode `json:"pay_mode,omitempty"`
+ Provider string `json:"provider,omitempty"`
+ ResourceType string `json:"resource_type,omitempty"`
+ RepoTag string `json:"repo_tag,omitempty"`
+}
+
+type RuntimeFrameworkReq struct {
+ FrameName string `json:"frame_name"`
+ FrameVersion string `json:"frame_version"`
+ FrameImage string `json:"frame_image"`
+ FrameCpuImage string `json:"frame_cpu_image"`
+ Enabled int64 `json:"enabled"`
+ ContainerPort int `json:"container_port"`
+ Type int `json:"type"`
+}
+
+type RuntimeFramework struct {
+ ID int64 `json:"id"`
+ FrameName string `json:"frame_name"`
+ FrameVersion string `json:"frame_version"`
+ FrameImage string `json:"frame_image"`
+ FrameCpuImage string `json:"frame_cpu_image"`
+ FrameNpuImage string `json:"frame_npu_image"`
+ Enabled int64 `json:"enabled"`
+ ContainerPort int `json:"container_port"`
+ Type int `json:"type"`
+}
+
+type RuntimeFrameworkModels struct {
+ Models []string `json:"models"`
+}
+
+type RepoFilter struct {
+ Tags []TagReq
+ Sort string
+ Search string
+ Source string
+ Username string
+}
+
+type TagReq struct {
+ Name string `json:"name"`
+ Category string `json:"category"`
+}
+
+type RuntimeArchitecture struct {
+ Architectures []string `json:"architectures"`
+}
+
+type ScanReq struct {
+ FrameID int64
+ FrameType int
+ ArchMap map[string]string
+ Models []string
+}
+
+type PermissionError struct {
+ Message string
+}
+
+// Add the Error() method to PermissionError.
+func (e *PermissionError) Error() string {
+ return e.Message // Return the message field as the error description.
+}
+
+
+
package types
+
+type CreateSSHKeyRequest struct {
+ Username string `json:"username"`
+ Name string `json:"name"`
+ Content string `json:"content"`
+}
+
+var _ SensitiveRequestV2 = (*CreateSSHKeyRequest)(nil)
+
+func (c *CreateSSHKeyRequest) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "name",
+ Value: func() string {
+ return c.Content
+ },
+ Scenario: "nickname_detection",
+ },
+ }
+}
+
+
+
package types
+
+import "time"
+
+type CreateUserRequest struct {
+ // Display name of the user
+ Name string `json:"name"`
+ // the login name
+ Username string `json:"username"`
+ Email string `json:"email" binding:"email"`
+ Phone string `json:"phone"`
+ UUID string `json:"uuid"`
+ // user registered from default login page, from casdoor, etc. Possible values:
+ //
+ // - "default"
+ // - "casdoor"
+ RegProvider string `json:"reg_provider"`
+}
+
+type UpdateUserRequest struct {
+ // Display name of the user
+ Nickname *string `json:"name,omitempty"`
+ // the login name
+ Username string `json:"-"`
+ Email *string `json:"email,omitempty" binding:"omitnil,email"`
+ Phone *string `json:"phone,omitempty"`
+ UUID *string `json:"uuid,omitempty"`
+ // should be updated by admin
+ Roles *[]string `json:"roles,omitempty" example:"[super_user, admin, personal_user]"`
+ Avatar *string `json:"avatar,omitempty"`
+ Homepage *string `json:"homepage,omitempty"`
+ Bio *string `json:"bio,omitempty"`
+
+ //if use want to change username, this should be the only field to send in request body
+ NewUserName *string `json:"new_username,omitempty"`
+}
+
+var _ SensitiveRequestV2 = (*UpdateUserRequest)(nil)
+
+func (u *UpdateUserRequest) GetSensitiveFields() []SensitiveField {
+ var fields []SensitiveField
+ if u.NewUserName != nil {
+ fields = append(fields, SensitiveField{
+ Name: "new_username",
+ Value: func() string {
+ return *u.NewUserName
+ },
+ Scenario: "nickname_detection",
+ })
+ }
+
+ if u.Nickname != nil {
+ fields = append(fields, SensitiveField{
+ Name: "nickname",
+ Value: func() string {
+ return *u.Nickname
+ },
+ Scenario: "nickname_detection",
+ })
+ }
+
+ if u.Bio != nil {
+ fields = append(fields, SensitiveField{
+ Name: "bio",
+ Value: func() string {
+ return *u.Bio
+ },
+ Scenario: "comment_detection",
+ })
+ }
+
+ if u.Homepage != nil {
+ fields = append(fields, SensitiveField{
+ Name: "homepage",
+ Value: func() string {
+ return *u.Homepage
+ },
+ Scenario: "chat_detection",
+ })
+ }
+ return fields
+}
+
+type UpdateUserResp struct {
+ Username string `json:"username"`
+ Email string `json:"email"`
+}
+
+type CreateUserTokenRequest struct {
+ Username string `json:"-" `
+ TokenName string `json:"name"`
+ // default to csghub
+ Application AccessTokenApp `json:"application,omitempty"`
+ // default to empty, means full permission
+ Permission string `json:"permission,omitempty"`
+ ExpiredAt time.Time `json:"expired_at"`
+}
+
+// CreateUserTokenRequest implements SensitiveRequestV2
+var _ SensitiveRequestV2 = (*CreateUserTokenRequest)(nil)
+
+func (c *CreateUserTokenRequest) GetSensitiveFields() []SensitiveField {
+ return []SensitiveField{
+ {
+ Name: "name",
+ Value: func() string {
+ return c.TokenName
+ },
+ Scenario: "nickname_detection",
+ },
+ }
+}
+
+type CheckAccessTokenReq struct {
+ Token string `json:"token" binding:"required"`
+ // Optional, if given only check the token belongs to this application
+ Application string `json:"application"`
+}
+
+type CheckAccessTokenResp struct {
+ Token string `json:"token"`
+ TokenName string `json:"token_name"`
+ Application AccessTokenApp `json:"application"`
+ Permission string `json:"permission,omitempty"`
+ // the login name
+ Username string `json:"user_name"`
+ UserUUID string `json:"user_uuid"`
+ ExpireAt time.Time `json:"expire_at"`
+}
+
+type UserDatasetsReq struct {
+ Owner string `json:"owner"`
+ CurrentUser string `json:"current_user"`
+ PageOpts
+}
+
+type (
+ UserModelsReq = UserDatasetsReq
+ UserCodesReq = UserDatasetsReq
+ UserSpacesReq = UserDatasetsReq
+ UserCollectionReq = UserDatasetsReq
+ DeleteUserTokenRequest = CreateUserTokenRequest
+ UserPromptsReq = UserDatasetsReq
+ UserEvaluationReq = UserDatasetsReq
+)
+
+type PageOpts struct {
+ Page int `json:"page"`
+ PageSize int `json:"page_size"`
+}
+
+type User struct {
+ ID int64 `json:"id,omitempty"`
+ Username string `json:"username"`
+ Nickname string `json:"nickname"`
+ Phone string `json:"phone,omitempty"`
+ Email string `json:"email,omitempty"`
+ UUID string `json:"uuid,omitempty"`
+ Avatar string `json:"avatar,omitempty"`
+ Bio string `json:"bio,omitempty"`
+ Homepage string `json:"homepage,omitempty"`
+ Roles []string `json:"roles,omitempty"`
+ LastLoginAt string `json:"last_login_at,omitempty"`
+ Orgs []Organization `json:"orgs,omitempty"`
+ CanChangeUserName bool `json:"can_change_username,omitempty"`
+}
+
+type UserLikesRequest struct {
+ Username string `json:"username"`
+ Repo_id int64 `json:"repo_id"`
+ Collection_id int64 `json:"collection_id"`
+ CurrentUser string `json:"current_user"`
+}
+
+/* for HF compatible apis */
+type WhoamiResponse struct {
+ Name string `json:"name"`
+ Email string `json:"email"`
+ Auth Auth `json:"auth"`
+}
+
+type AccessToken struct {
+ DisplayName string `json:"displayName,omitempty"`
+ Role string `json:"role,omitempty"`
+}
+
+type Auth struct {
+ AccessToken `json:"accessToken,omitempty"`
+ Type string `json:"type,omitempty"`
+}
+
+type UserRepoReq struct {
+ CurrentUser string `json:"current_user"`
+ PageOpts
+}
+
+type AccessTokenApp string
+
+const (
+ AccessTokenAppGit AccessTokenApp = "git"
+ AccessTokenAppCSGHub = AccessTokenAppGit
+ AccessTokenAppMirror AccessTokenApp = "mirror"
+ AccessTokenAppStarship AccessTokenApp = "starship"
+)
+
+type UserRepoPermission struct {
+ CanRead bool `json:"can_read"`
+ CanWrite bool `json:"can_write"`
+ CanAdmin bool `json:"can_admin"`
+}
+
+
+
package common
+
+import (
+ "crypto/sha256"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "golang.org/x/crypto/ssh"
+ "opencsg.com/csghub-server/common/types"
+)
+
+func GetNamespaceAndNameFromContext(ctx *gin.Context) (namespace string, name string, err error) {
+ namespace = ctx.Param("namespace")
+ name = ctx.Param("name")
+ namespace_mapped := ctx.GetString("namespace_mapped")
+ if namespace_mapped != "" {
+ namespace = namespace_mapped
+ }
+ name_mapped := ctx.GetString("name_mapped")
+ if name_mapped != "" {
+ name = name_mapped
+ }
+ if namespace == "" || name == "" {
+ err = errors.New("invalid namespace or name")
+ return
+ }
+ return
+}
+
+func GetPerAndPageFromContext(ctx *gin.Context) (perInt int, pageInt int, err error) {
+ per := ctx.Query("per")
+ if per == "" {
+ per = "50"
+ }
+ perInt, err = strconv.Atoi(per)
+ if err != nil {
+ return
+ }
+ page := ctx.Query("page")
+ if page == "" {
+ page = "1"
+ }
+ pageInt, err = strconv.Atoi(page)
+ if err != nil {
+ return
+ }
+ return
+}
+
+func RepoTypeFromContext(ctx *gin.Context) types.RepositoryType {
+ rawRp, exist := ctx.Get("repo_type")
+ slog.Debug("get repo type from context", "repo_type", rawRp, "exists", exist)
+ if !exist {
+ return types.UnknownRepo
+ }
+ return rawRp.(types.RepositoryType)
+}
+
+func SetRepoTypeContext(ctx *gin.Context, t types.RepositoryType) {
+ ctx.Set("repo_type", t)
+}
+
+func RepoTypeFromParam(ctx *gin.Context) types.RepositoryType {
+ rawRp := ctx.Param("repo_type")
+ slog.Debug("get repo type from parameters", "repo_type", rawRp)
+ return types.RepositoryType(rawRp)
+}
+
+func CalculateSSHKeyFingerprint(key string) (string, error) {
+ parsedKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(key))
+ if err != nil {
+ return "", err
+ }
+ fingerPrint := ssh.FingerprintSHA256(parsedKey)
+ fingerPrint = strings.Split(fingerPrint, ":")[1]
+ return fingerPrint, nil
+}
+
+func CalculateAuthorizedSSHKeyFingerprint(key string) (string, error) {
+ decodedKey, err := base64.RawStdEncoding.DecodeString(key)
+ if err != nil {
+ return "", fmt.Errorf("base64 decode error: %w", err)
+ }
+
+ hash := sha256.Sum256(decodedKey)
+ base64Hash := base64.RawStdEncoding.EncodeToString(hash[:])
+ return base64Hash, nil
+}
+
+
+
package common
+
+import (
+ "fmt"
+ "net/url"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+const (
+ ModelOrgPrefix = "models_"
+ DatasetOrgPrefix = "datasets_"
+ SpaceOrgPrefix = "spaces_"
+ CodeOrgPrefix = "codes_"
+)
+
+func WithPrefix(name string, prefix string) string {
+ return prefix + name
+}
+
+func WithoutPrefix(name string, prefix string) string {
+ return strings.Replace(name, prefix, "", 1)
+}
+
+func ConvertDotToSlash(d string) string {
+ if d == "." {
+ return "/"
+ } else {
+ return d
+ }
+}
+
+func PortalCloneUrl(url string, repoType types.RepositoryType, gitDomain, portalDomain string) string {
+ prefix := repoPrefixByType(repoType)
+ url = strings.Replace(url, prefix, fmt.Sprintf("%s/", prefix[:len(prefix)-1]), 1)
+ url = strings.Replace(url, gitDomain, portalDomain, 1)
+ return url
+}
+
+func repoPrefixByType(repoType types.RepositoryType) string {
+ var prefix string
+ switch repoType {
+ case types.ModelRepo:
+ prefix = ModelOrgPrefix
+ case types.DatasetRepo:
+ prefix = DatasetOrgPrefix
+ case types.SpaceRepo:
+ prefix = SpaceOrgPrefix
+ case types.CodeRepo:
+ prefix = CodeOrgPrefix
+ }
+
+ return prefix
+}
+
+func BuildCloneInfo(config *config.Config, repository *database.Repository) types.Repository {
+ return types.Repository{
+ HTTPCloneURL: buildHTTPCloneURL(config.APIServer.PublicDomain, repository.RepositoryType, repository.Path),
+ SSHCloneURL: buildSSHCloneURL(config.APIServer.SSHDomain, repository.RepositoryType, repository.Path),
+ }
+}
+
+func buildHTTPCloneURL(domain string, repoType types.RepositoryType, path string) string {
+ return fmt.Sprintf("%s/%ss/%s.git", strings.TrimSuffix(domain, "/"), repoType, path)
+}
+
+func buildSSHCloneURL(domain string, repoType types.RepositoryType, path string) string {
+ parsedURL, err := url.Parse(domain)
+ if err != nil {
+ return ""
+ }
+ sshDomainWithoutPrefix := strings.TrimPrefix(domain, "ssh://")
+
+ if parsedURL.Port() == "" {
+ return fmt.Sprintf("%s:%ss/%s.git", strings.TrimSuffix(sshDomainWithoutPrefix, "/"), repoType, path)
+ } else {
+ return fmt.Sprintf("ssh://%s/%ss/%s.git", strings.TrimSuffix(sshDomainWithoutPrefix, "/"), repoType, path)
+ }
+}
+
+
+
package common
+
+// TruncString return a substring if the input string is larger than limit size, truncated string ends with "..."
+func TruncString(s string, limit int) string {
+ if len(s) <= limit {
+ return s
+ }
+
+ s1 := []byte(s[:limit])
+ s1[limit-1] = '.'
+ s1[limit-2] = '.'
+ s1[limit-3] = '.'
+ return string(s1)
+}
+
+
+
package common
+
+import (
+ "fmt"
+ "strings"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+func AddPrefixBySourceID(sourceID int64, originString string) string {
+ return fmt.Sprintf("%s%s", getprefixBySourceID(sourceID), originString)
+}
+
+func TrimPrefixCloneURLBySourceID(url, repoType, namespace, name string, sourceID int64) string {
+ namespace, _ = strings.CutPrefix(namespace, getprefixBySourceID(sourceID))
+ return fmt.Sprintf("%s/%ss/%s/%s.git", url, repoType, namespace, name)
+}
+
+func getprefixBySourceID(sourceID int64) string {
+ var prefix string
+ if sourceID == int64(types.SyncVersionSourceOpenCSG) {
+ prefix = types.OpenCSGPrefix
+ } else if sourceID == int64(types.SyncVersionSourceHF) {
+ prefix = types.HuggingfacePrefix
+ }
+ return prefix
+}
+
+
+
package utils
+
+// UnwrapError recursively.
+func UnwrapError(err error) error {
+ for err != nil {
+ if wrappedErr, ok := err.(interface{ Unwrap() error }); ok {
+ err = wrappedErr.Unwrap()
+ } else {
+ break
+ }
+ }
+
+ return err
+}
+
+
+
package money
+
+type Currency string
+
+const (
+ CurrencyCNY Currency = "CNY" // Chinese Yuan
+ CurrencyUSD Currency = "USD" // US Dollar
+ CurrencyEUR Currency = "EUR" // Euro
+ CurrencyJPY Currency = "JPY" // Japanese Yen
+ CurrencyGBP Currency = "GBP" // British Pound
+)
+
+func isValidCurrency(currency Currency) bool {
+ switch currency {
+ case CurrencyCNY, CurrencyUSD, CurrencyEUR, CurrencyJPY, CurrencyGBP:
+ return true
+ default:
+ return false
+ }
+}
+
+
+
package money
+
+import (
+ "errors"
+ "fmt"
+ "math/big"
+)
+
+// Money struct to handle currency values safely
+type Money struct {
+ amount int64 // Amount stored in the smallest unit (e.g., cents)
+ currency Currency // 3-letter ISO currency code (e.g., "CNY", "USD")
+}
+
+// NewMoney creates a new Money instance
+func NewMoney(amount int64, currency Currency) (*Money, error) {
+ if !isValidCurrency(currency) {
+ return nil, errors.New("invalid currency")
+ }
+ return &Money{
+ amount: amount,
+ currency: currency,
+ }, nil
+}
+
+// NewMoneyFromYuan converts a yuan amount into a Money object in fen.
+//
+// Parameters:
+// - yuanAmount (float64): The amount in yuan to convert.
+//
+// Returns:
+// - *Money: A Money instance representing the amount in fen.
+// - error: An error if the conversion fails.
+//
+// Note:
+// - This method assumes that 1 yuan equals 100 fen.
+// - Be cautious with floating-point precision when dealing with currency amounts.
+func NewMoneyFromYuan(yuanAmount float64) (*Money, error) {
+ fenAmount := int64(yuanAmount * 100)
+ return NewMoney(fenAmount, CurrencyCNY)
+}
+
+func (m *Money) validateSameCurrency(other *Money) error {
+ if m.currency != other.currency {
+ return fmt.Errorf("currency mismatch: %s vs %s", m.currency, other.currency)
+ }
+ return nil
+}
+
+// Add adds another Money value (must have the same currency)
+func (m *Money) Add(other *Money) (*Money, error) {
+ if err := m.validateSameCurrency(other); err != nil {
+ return nil, err
+ }
+ newAmount := m.amount + other.amount
+ return &Money{amount: newAmount, currency: m.currency}, nil
+}
+
+// Sub subtracts another Money value (must have the same currency)
+func (m *Money) Sub(other *Money) (*Money, error) {
+ if err := m.validateSameCurrency(other); err != nil {
+ return nil, err
+ }
+ newAmount := m.amount - other.amount
+ return &Money{amount: newAmount, currency: m.currency}, nil
+}
+
+// Multiply multiplies the Money amount by a scalar
+func (m *Money) Multiply(factor int64) *Money {
+ newAmount := m.amount * factor
+ return &Money{amount: newAmount, currency: m.currency}
+}
+
+// Divide divides the Money amount by a scalar
+func (m *Money) Divide(divisor int64) (*Money, error) {
+ if divisor == 0 {
+ return nil, fmt.Errorf("cannot divide by zero")
+ }
+ newAmount := m.amount / divisor
+ return &Money{amount: newAmount, currency: m.currency}, nil
+}
+
+// Format returns the formatted string representation of Money
+func (m *Money) Format() string {
+ // Convert to the "major" unit (e.g., dollars from cents)
+ major := big.NewRat(m.amount, 100)
+ return fmt.Sprintf("%s %s", major.FloatString(2), m.currency)
+}
+
+// GetAmount returns the amount in the smallest unit (e.g., cents)
+func (m *Money) GetAmount() int64 {
+ return m.amount
+}
+
+// GetCurrency returns the currency code
+func (m *Money) GetCurrency() Currency {
+ return m.currency
+}
+
+// toYuanRat is an internal helper method that converts the Money amount to a big.Rat representing yuan.
+// It validates that the currency is CNY and checks if the amount is within Alipay's required range.
+//
+// Returns:
+// - *big.Rat: A big.Rat representing the amount in yuan.
+// - error: An error if the currency is not CNY or the amount is out of range.
+func (m *Money) toYuanRat() (*big.Rat, error) {
+ // Validate that the currency is CNY
+ if m.currency != CurrencyCNY {
+ return nil, fmt.Errorf("currency mismatch: expected CNY, got %s", m.currency)
+ }
+
+ // Create a big.Rat representing the amount in fen
+ amountRat := new(big.Rat).SetInt64(m.amount)
+
+ // Conversion factor: 1 yuan = 100 fen
+ factor := big.NewRat(1, 100)
+
+ // Convert from fen to yuan
+ yuanRat := new(big.Rat).Mul(amountRat, factor)
+
+ // Alipay requires the amount to be within [0.01, 100000000]
+ minAmount := big.NewRat(1, 100) // 0.01
+ maxAmount := big.NewRat(100000000, 1) // 100000000
+
+ if yuanRat.Cmp(minAmount) < 0 || yuanRat.Cmp(maxAmount) > 0 {
+ return nil, fmt.Errorf("amount %s is out of allowed range [0.01, 100000000]", yuanRat.FloatString(2))
+ }
+
+ return yuanRat, nil
+}
+
+// ToYuanString converts the Money amount to a string representing yuan, formatted to two decimal places.
+func (m *Money) ToYuanString() (string, error) {
+ yuanRat, err := m.toYuanRat()
+ if err != nil {
+ return "", err
+ }
+ // Format the big.Rat to a string with two decimal places
+ return yuanRat.FloatString(2), nil
+}
+
+// ToYuanFloat converts the Money amount to a float64 representing yuan.
+func (m *Money) ToYuanFloat() (float64, error) {
+ yuanRat, err := m.toYuanRat()
+ if err != nil {
+ return 0, err
+ }
+ // Convert the big.Rat to a float64
+ yuanFloat, _ := yuanRat.Float64()
+ return yuanFloat, nil
+}
+
+
+
package payment
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/bwmarrin/snowflake"
+)
+
+func GenerateOrderNumber() (string, error) {
+ number, err := GenerateOrderNumberBySnowFlake(1)
+ return number, err
+}
+
+func GenerateOrderNumberBySnowFlake(nodeNum int64) (string, error) {
+ node, err := snowflake.NewNode(nodeNum)
+ if err != nil {
+ return "", err
+ }
+ id := node.Generate()
+
+ date := time.Now().Format("20060102")
+
+ orderNumber := fmt.Sprintf("%s%d", date, id)
+
+ return orderNumber, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/accounting"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type accountingComponentImpl struct {
+ acctountingClient accounting.AccountingClient
+ userStore database.UserStore
+ deployTaskStore database.DeployTaskStore
+}
+
+type AccountingComponent interface {
+ QueryAllUsersBalance(ctx context.Context, currentUser string, per, page int) (interface{}, error)
+ QueryBalanceByUserID(ctx context.Context, currentUser, userUUID string) (interface{}, error)
+ QueryBalanceByUserIDInternal(ctx context.Context, currentUser string) (*database.AccountUser, error)
+ ListStatementByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (interface{}, error)
+ ListBillsByUserIDAndDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (interface{}, error)
+ RechargeAccountingUser(ctx context.Context, currentUser, userUUID string, req types.RECHARGE_REQ) (interface{}, error)
+ CreateOrUpdateQuota(currentUser string, req types.ACCT_QUOTA_REQ) (interface{}, error)
+ GetQuotaByID(currentUser string) (interface{}, error)
+ CreateQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (interface{}, error)
+ GetQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (interface{}, error)
+ QueryPricesBySKUType(currentUser string, req types.AcctPriceListReq) (*database.PriceResp, error)
+ GetPriceByID(currentUser string, id int64) (interface{}, error)
+ CreatePrice(currentUser string, req types.AcctPriceCreateReq) (interface{}, error)
+ UpdatePrice(currentUser string, req types.AcctPriceCreateReq, id int64) (interface{}, error)
+ DeletePrice(currentUser string, id int64) (interface{}, error)
+ CreateOrder(currentUser string, req types.AcctOrderCreateReq) (*database.AccountOrder, error)
+ ListMeteringsByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (interface{}, error)
+}
+
+func NewAccountingComponent(config *config.Config) (AccountingComponent, error) {
+ c, err := accounting.NewAccountingClient(config)
+ if err != nil {
+ return nil, err
+ }
+ return &accountingComponentImpl{
+ acctountingClient: c,
+ userStore: database.NewUserStore(),
+ deployTaskStore: database.NewDeployTaskStore(),
+ }, nil
+}
+
+func (ac *accountingComponentImpl) QueryAllUsersBalance(ctx context.Context, currentUser string, per, page int) (interface{}, error) {
+ user, err := ac.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ // only admin can query all users
+ if !user.CanAdmin() {
+ return nil, fmt.Errorf("only admin was allowed to query all users balance")
+ }
+ return ac.acctountingClient.QueryAllUsersBalance(per, page)
+}
+
+func (ac *accountingComponentImpl) QueryBalanceByUserID(ctx context.Context, currentUser, userUUID string) (interface{}, error) {
+ user, err := ac.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ // only admin can query other user's balance
+ if !user.CanAdmin() && user.UUID != userUUID {
+ return nil, errors.New("invalid user to query balance")
+ }
+ return ac.acctountingClient.QueryBalanceByUserID(userUUID)
+}
+
+func (ac *accountingComponentImpl) QueryBalanceByUserIDInternal(ctx context.Context, currentUser string) (*database.AccountUser, error) {
+ user, err := ac.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ resp, err := ac.acctountingClient.QueryBalanceByUserID(user.UUID)
+ if err != nil {
+ return nil, fmt.Errorf("error to get user balance data, %w", err)
+ }
+
+ tempJSON, err := json.Marshal(resp)
+ if err != nil {
+ return nil, fmt.Errorf("error to marshal json, %w", err)
+ }
+ var account *database.AccountUser
+ if err := json.Unmarshal(tempJSON, &account); err != nil {
+ return nil, fmt.Errorf("error to unmarshal json, %w", err)
+ }
+ return account, nil
+}
+
+func (ac *accountingComponentImpl) ListStatementByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (interface{}, error) {
+ user, err := ac.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ if user.UUID != req.UserUUID {
+ return nil, errors.New("invalid user")
+ }
+ return ac.acctountingClient.ListStatementByUserIDAndTime(req)
+}
+
+func (ac *accountingComponentImpl) ListBillsByUserIDAndDate(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (interface{}, error) {
+ user, err := ac.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ if user.UUID != req.UserUUID {
+ return nil, errors.New("invalid user")
+ }
+
+ data, err := ac.acctountingClient.ListBillsByUserIDAndDate(req)
+ if err != nil {
+ return nil, err
+ }
+ bills, err := ac.parseBillsData(data)
+ // slog.Info("convert", slog.Any("data", data), slog.Any("bills", bills))
+ if err != nil {
+ return nil, err
+ }
+ if bills == nil || bills.Data == nil {
+ return bills, nil
+ }
+ var mergedItems []types.ITEM
+ for _, item := range bills.Data {
+ newItem := types.ITEM{
+ Consumption: item.Consumption,
+ InstanceName: item.InstanceName,
+ Value: item.Value,
+ }
+ d, _ := ac.deployTaskStore.GetDeployBySvcName(ctx, item.InstanceName)
+ if d != nil {
+ newItem.Status = deployStatusCodeToString(d.Status)
+ newItem.CreatedAt = d.CreatedAt
+ newItem.DeployID = d.ID
+ newItem.DeployName = d.DeployName
+ newItem.DeployUser = req.CurrentUser
+ if d.GitPath != "" {
+ idx := strings.Index(d.GitPath, "_")
+ if idx > -1 && idx+1 < len(d.GitPath) {
+ newItem.RepoPath = d.GitPath[idx+1:]
+ }
+ }
+ }
+ mergedItems = append(mergedItems, newItem)
+ }
+ return types.BILLS{
+ Data: mergedItems,
+ ACCT_SUMMARY: types.ACCT_SUMMARY{
+ Total: bills.Total,
+ TotalValue: bills.TotalValue,
+ TotalConsumption: bills.TotalConsumption,
+ },
+ }, err
+}
+
+func (ac *accountingComponentImpl) RechargeAccountingUser(ctx context.Context, currentUser, userUUID string, req types.RECHARGE_REQ) (interface{}, error) {
+ user, err := ac.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ _, err = ac.userStore.FindByUUID(ctx, userUUID)
+ if err != nil {
+ return nil, fmt.Errorf("invalid user uuid, %w", err)
+ }
+ // only admin can recharge
+ if !user.CanAdmin() {
+ return nil, fmt.Errorf("only admin was allowed to recharge")
+ }
+ return ac.acctountingClient.RechargeAccountingUser(userUUID, req)
+}
+
+func (ac *accountingComponentImpl) CreateOrUpdateQuota(currentUser string, req types.ACCT_QUOTA_REQ) (interface{}, error) {
+ return ac.acctountingClient.CreateOrUpdateQuota(currentUser, req)
+}
+
+func (ac *accountingComponentImpl) GetQuotaByID(currentUser string) (interface{}, error) {
+ return ac.acctountingClient.GetQuotaByID(currentUser)
+}
+
+func (ac *accountingComponentImpl) CreateQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (interface{}, error) {
+ return ac.acctountingClient.CreateQuotaStatement(currentUser, req)
+}
+
+func (ac *accountingComponentImpl) GetQuotaStatement(currentUser string, req types.ACCT_QUOTA_STATEMENT_REQ) (interface{}, error) {
+ return ac.acctountingClient.GetQuotaStatement(currentUser, req)
+}
+
+func (ac *accountingComponentImpl) parseBillsData(data interface{}) (*types.BILLS, error) {
+ resByte, err := json.Marshal(data)
+ if err != nil {
+ return nil, err
+ }
+ var newData types.BILLS
+ err = json.Unmarshal(resByte, &newData)
+ if err != nil {
+ return nil, err
+ }
+ return &newData, nil
+}
+
+func (ac *accountingComponentImpl) QueryPricesBySKUType(currentUser string, req types.AcctPriceListReq) (*database.PriceResp, error) {
+ resp, err := ac.acctountingClient.QueryPricesBySKUType(currentUser, req)
+ if err != nil {
+ return nil, fmt.Errorf("error to get price, %w", err)
+ }
+ tempJSON, err := json.Marshal(resp)
+ if err != nil {
+ return nil, fmt.Errorf("error to marshal json, %w", err)
+ }
+ var priceData database.PriceResp
+ if err := json.Unmarshal(tempJSON, &priceData); err != nil {
+ return nil, fmt.Errorf("error to unmarshal json, %w", err)
+ }
+ return &priceData, nil
+}
+
+func (ac *accountingComponentImpl) GetPriceByID(currentUser string, id int64) (interface{}, error) {
+ return ac.acctountingClient.GetPriceByID(currentUser, id)
+}
+
+func (ac *accountingComponentImpl) CreatePrice(currentUser string, req types.AcctPriceCreateReq) (interface{}, error) {
+ return ac.acctountingClient.CreatePrice(currentUser, req)
+}
+
+func (ac *accountingComponentImpl) UpdatePrice(currentUser string, req types.AcctPriceCreateReq, id int64) (interface{}, error) {
+ return ac.acctountingClient.UpdatePrice(currentUser, req, id)
+}
+
+func (ac *accountingComponentImpl) DeletePrice(currentUser string, id int64) (interface{}, error) {
+ return ac.acctountingClient.DeletePrice(currentUser, id)
+}
+
+func (ac *accountingComponentImpl) CreateOrder(currentUser string, req types.AcctOrderCreateReq) (*database.AccountOrder, error) {
+ resp, err := ac.acctountingClient.CreateOrder(currentUser, req)
+ if err != nil {
+ return nil, fmt.Errorf("error to create order, %w", err)
+ }
+
+ tempJSON, err := json.Marshal(resp)
+ if err != nil {
+ return nil, fmt.Errorf("error to marshal json, %w", err)
+ }
+ var order *database.AccountOrder
+ if err := json.Unmarshal(tempJSON, &order); err != nil {
+ return nil, fmt.Errorf("error to unmarshal json, %w", err)
+ }
+ return order, nil
+}
+
+func (ac *accountingComponentImpl) ListMeteringsByUserIDAndTime(ctx context.Context, req types.ACCT_STATEMENTS_REQ) (interface{}, error) {
+ user, err := ac.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ if user.UUID != req.UserUUID {
+ return nil, errors.New("invalid user")
+ }
+ return ac.acctountingClient.ListMeteringsByUserIDAndTime(req)
+}
+
+
+
package callback
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "path"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+// define GitCallbackComponent struct
+type GitCallbackComponent struct {
+ config *config.Config
+ gs gitserver.GitServer
+ tc component.TagComponent
+ modSvcClient rpc.ModerationSvcClient
+ ms database.ModelStore
+ ds database.DatasetStore
+ sc component.SpaceComponent
+ ss database.SpaceStore
+ rs database.RepoStore
+ rrs database.RepoRelationsStore
+ mirrorStore database.MirrorStore
+ svGen *SyncVersionGenerator
+ rrf database.RepositoriesRuntimeFrameworkStore
+ rac component.RuntimeArchitectureComponent
+ ras database.RuntimeArchitecturesStore
+ rfs database.RuntimeFrameworksStore
+ ts database.TagStore
+ dt database.TagRuleStore
+ // set visibility if file content is sensitive
+ setRepoVisibility bool
+ pp component.PromptComponent
+ maxPromptFS int64
+}
+
+// new CallbackComponent
+func NewGitCallback(config *config.Config) (*GitCallbackComponent, error) {
+ gs, err := git.NewGitServer(config)
+ if err != nil {
+ return nil, err
+ }
+ tc, err := component.NewTagComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ ms := database.NewModelStore()
+ ds := database.NewDatasetStore()
+ ss := database.NewSpaceStore()
+ rs := database.NewRepoStore()
+ rrs := database.NewRepoRelationsStore()
+ mirrorStore := database.NewMirrorStore()
+ sc, err := component.NewSpaceComponent(config)
+ ras := database.NewRuntimeArchitecturesStore()
+ if err != nil {
+ return nil, err
+ }
+ svGen := NewSyncVersionGenerator()
+ rrf := database.NewRepositoriesRuntimeFramework()
+ rac, err := component.NewRuntimeArchitectureComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ rfs := database.NewRuntimeFrameworksStore()
+ ts := database.NewTagStore()
+ pp, err := component.NewPromptComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ var modSvcClient rpc.ModerationSvcClient
+ if config.SensitiveCheck.Enable {
+ modSvcClient = rpc.NewModerationSvcHttpClient(fmt.Sprintf("%s:%d", config.Moderation.Host, config.Moderation.Port))
+ }
+ dt := database.NewTagRuleStore()
+ return &GitCallbackComponent{
+ config: config,
+ gs: gs,
+ tc: tc,
+ ms: ms,
+ ds: ds,
+ ss: ss,
+ sc: sc,
+ rs: rs,
+ rrs: rrs,
+ mirrorStore: mirrorStore,
+ modSvcClient: modSvcClient,
+ svGen: svGen,
+ rrf: rrf,
+ rac: rac,
+ ras: ras,
+ rfs: rfs,
+ pp: pp,
+ ts: ts,
+ dt: dt,
+ maxPromptFS: config.Dataset.PromptMaxJsonlFileSize,
+ }, nil
+}
+
+// SetRepoVisibility sets a flag whether change repo's visibility if file content is sensitive
+func (c *GitCallbackComponent) SetRepoVisibility(yes bool) {
+ c.setRepoVisibility = yes
+}
+
+func (c *GitCallbackComponent) WatchSpaceChange(ctx context.Context, req *types.GiteaCallbackPushReq) error {
+ err := WatchSpaceChange(req, c.ss, c.sc).Run()
+ if err != nil {
+ slog.Error("watch space change failed", slog.Any("error", err))
+ return err
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) WatchRepoRelation(ctx context.Context, req *types.GiteaCallbackPushReq) error {
+ err := WatchRepoRelation(req, c.rs, c.rrs, c.gs).Run()
+ if err != nil {
+ slog.Error("watch repo relation failed", slog.Any("error", err))
+ return err
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) GenSyncVersion(ctx context.Context, req *types.GiteaCallbackPushReq) error {
+ if !req.Repository.Private {
+ err := c.svGen.GenSyncVersion(req)
+ if err != nil {
+ slog.Error("generate sync version failed", slog.Any("error", err))
+ return err
+ }
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) SetRepoUpdateTime(ctx context.Context, req *types.GiteaCallbackPushReq) error {
+ // split req.Repository.FullName by '/'
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+ adjustedRepoType := types.RepositoryType(strings.TrimRight(repoType, "s"))
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+
+ isMirrorRepo, err := c.rs.IsMirrorRepo(ctx, adjustedRepoType, namespace, repoName)
+ if err != nil {
+ slog.Error("failed to check if a mirror repo", slog.Any("error", err), slog.String("repo_type", string(adjustedRepoType)), slog.String("namespace", namespace), slog.String("name", repoName))
+ return err
+ }
+ if isMirrorRepo {
+ updated, err := time.Parse(time.RFC3339, req.HeadCommit.Timestamp)
+ if err != nil {
+ slog.Error("Error parsing time:", slog.Any("error", err), slog.String("timestamp", req.HeadCommit.Timestamp))
+ return err
+ }
+ err = c.rs.SetUpdateTimeByPath(ctx, adjustedRepoType, namespace, repoName, updated)
+ if err != nil {
+ slog.Error("failed to set repo update time", slog.Any("error", err), slog.String("repo_type", string(adjustedRepoType)), slog.String("namespace", namespace), slog.String("name", repoName))
+ return err
+ }
+ mirror, err := c.mirrorStore.FindByRepoPath(ctx, adjustedRepoType, namespace, repoName)
+ if err != nil {
+ slog.Error("failed to find repo mirror", slog.Any("error", err), slog.String("repo_type", string(adjustedRepoType)), slog.String("namespace", namespace), slog.String("name", repoName))
+ return err
+ }
+ mirror.LastUpdatedAt = time.Now()
+ err = c.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ slog.Error("failed to update repo mirror last_updated_at", slog.Any("error", err), slog.String("repo_type", string(adjustedRepoType)), slog.String("namespace", namespace), slog.String("name", repoName))
+ return err
+ }
+ } else {
+ err := c.rs.SetUpdateTimeByPath(ctx, adjustedRepoType, namespace, repoName, time.Now())
+ if err != nil {
+ slog.Error("failed to set repo update time", slog.Any("error", err), slog.String("repo_type", string(adjustedRepoType)), slog.String("namespace", namespace), slog.String("name", repoName))
+ return err
+ }
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) UpdateRepoInfos(ctx context.Context, req *types.GiteaCallbackPushReq) error {
+ commits := req.Commits
+ ref := req.Ref
+ // split req.Repository.FullName by '/'
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+
+ var err error
+ for _, commit := range commits {
+ err = errors.Join(err, c.modifyFiles(ctx, repoType, namespace, repoName, ref, commit.Modified))
+ err = errors.Join(err, c.removeFiles(ctx, repoType, namespace, repoName, ref, commit.Removed))
+ err = errors.Join(err, c.addFiles(ctx, repoType, namespace, repoName, ref, commit.Added))
+ }
+
+ return err
+}
+
+func (c *GitCallbackComponent) SensitiveCheck(ctx context.Context, req *types.GiteaCallbackPushReq) error {
+ // split req.Repository.FullName by '/'
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+ adjustedRepoType := types.RepositoryType(strings.TrimRight(repoType, "s"))
+
+ var err error
+ if c.modSvcClient != nil {
+ err = c.modSvcClient.SubmitRepoCheck(ctx, adjustedRepoType, namespace, repoName)
+ }
+ if err != nil {
+ slog.Error("fail to submit repo sensitive check", slog.Any("error", err), slog.Any("repo_type", adjustedRepoType), slog.String("namespace", namespace), slog.String("name", repoName))
+ return err
+ }
+
+ return nil
+}
+
+// modifyFiles method handles modified files, skip if not modify README.md
+func (c *GitCallbackComponent) modifyFiles(ctx context.Context, repoType, namespace, repoName, ref string, fileNames []string) error {
+ for _, fileName := range fileNames {
+ slog.Debug("modify file", slog.String("file", fileName))
+ // update model runtime
+ c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false, fileNames)
+ // only care about readme file under root directory
+ if fileName != types.ReadmeFileName {
+ continue
+ }
+
+ content, err := c.getFileRaw(repoType, namespace, repoName, ref, fileName)
+ if err != nil {
+ return err
+ }
+ // should be only one README.md
+ return c.updateMetaTags(ctx, repoType, namespace, repoName, ref, content)
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) removeFiles(ctx context.Context, repoType, namespace, repoName, ref string, fileNames []string) error {
+ // handle removed files
+ // delete tags
+ for _, fileName := range fileNames {
+ slog.Debug("remove file", slog.String("file", fileName))
+ // update model runtime
+ c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, true, fileNames)
+ // only care about readme file under root directory
+ if fileName == types.ReadmeFileName {
+ // use empty content to clear all the meta tags
+ const content string = ""
+ adjustedRepoType := types.RepositoryType(strings.TrimSuffix(repoType, "s"))
+ err := c.tc.ClearMetaTags(ctx, adjustedRepoType, namespace, repoName)
+ if err != nil {
+ slog.Error("failed to clear meta tags", slog.String("content", content),
+ slog.String("repo", path.Join(namespace, repoName)), slog.String("ref", ref),
+ slog.Any("error", err))
+ return fmt.Errorf("failed to clear met tags,cause: %w", err)
+ }
+ } else {
+ var tagScope database.TagScope
+ switch repoType {
+ case fmt.Sprintf("%ss", types.DatasetRepo):
+ tagScope = database.DatasetTagScope
+ case fmt.Sprintf("%ss", types.ModelRepo):
+ tagScope = database.ModelTagScope
+ case fmt.Sprintf("%ss", types.PromptRepo):
+ tagScope = database.PromptTagScope
+ default:
+ return nil
+ // case CodeRepoType:
+ // tagScope = database.CodeTagScope
+ // case SpaceRepoType:
+ // tagScope = database.SpaceTagScope
+ }
+ err := c.tc.UpdateLibraryTags(ctx, tagScope, namespace, repoName, fileName, "")
+ if err != nil {
+ slog.Error("failed to remove Library tag", slog.String("namespace", namespace),
+ slog.String("name", repoName), slog.String("ref", ref), slog.String("fileName", fileName),
+ slog.Any("error", err))
+ return fmt.Errorf("failed to remove Library tag, cause: %w", err)
+ }
+ }
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) addFiles(ctx context.Context, repoType, namespace, repoName, ref string, fileNames []string) error {
+ for _, fileName := range fileNames {
+ slog.Debug("add file", slog.String("file", fileName))
+ // update model runtime
+ c.updateRepoRelations(ctx, repoType, namespace, repoName, ref, fileName, false, fileNames)
+ // only care about readme file under root directory
+ if fileName == types.ReadmeFileName {
+ content, err := c.getFileRaw(repoType, namespace, repoName, ref, fileName)
+ if err != nil {
+ return err
+ }
+ err = c.updateMetaTags(ctx, repoType, namespace, repoName, ref, content)
+ if err != nil {
+ return err
+ }
+ } else {
+ var tagScope database.TagScope
+ switch repoType {
+ case fmt.Sprintf("%ss", types.DatasetRepo):
+ tagScope = database.DatasetTagScope
+ case fmt.Sprintf("%ss", types.ModelRepo):
+ tagScope = database.ModelTagScope
+ case fmt.Sprintf("%ss", types.PromptRepo):
+ tagScope = database.PromptTagScope
+ default:
+ return nil
+ // case CodeRepoType:
+ // tagScope = database.CodeTagScope
+ // case SpaceRepoType:
+ // tagScope = database.SpaceTagScope
+ }
+ err := c.tc.UpdateLibraryTags(ctx, tagScope, namespace, repoName, "", fileName)
+ if err != nil {
+ slog.Error("failed to add Library tag", slog.String("namespace", namespace),
+ slog.String("name", repoName), slog.String("ref", ref), slog.String("fileName", fileName),
+ slog.Any("error", err))
+ return fmt.Errorf("failed to add Library tag, cause: %w", err)
+ }
+ }
+ }
+ return nil
+}
+
+func (c *GitCallbackComponent) updateMetaTags(ctx context.Context, repoType, namespace, repoName, ref, content string) error {
+ var (
+ err error
+ tagScope database.TagScope
+ )
+ switch repoType {
+ case fmt.Sprintf("%ss", types.DatasetRepo):
+ tagScope = database.DatasetTagScope
+ case fmt.Sprintf("%ss", types.ModelRepo):
+ tagScope = database.ModelTagScope
+ case fmt.Sprintf("%ss", types.PromptRepo):
+ tagScope = database.PromptTagScope
+ default:
+ return nil
+ // TODO: support code and space
+ // case CodeRepoType:
+ // tagScope = database.CodeTagScope
+ // case SpaceRepoType:
+ // tagScope = database.SpaceTagScope
+ }
+ _, err = c.tc.UpdateMetaTags(ctx, tagScope, namespace, repoName, content)
+ if err != nil {
+ slog.Error("failed to update meta tags", slog.String("namespace", namespace),
+ slog.String("content", content), slog.String("repo", repoName), slog.String("ref", ref),
+ slog.Any("error", err))
+ return fmt.Errorf("failed to update met tags,cause: %w", err)
+ }
+ slog.Info("update meta tags success", slog.String("repo", path.Join(namespace, repoName)), slog.String("type", repoType))
+ return nil
+}
+
+func (c *GitCallbackComponent) getFileRaw(repoType, namespace, repoName, ref, fileName string) (string, error) {
+ var (
+ content string
+ err error
+ )
+ repoType = strings.TrimRight(repoType, "s")
+ getFileRawReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: repoName,
+ Ref: ref,
+ Path: fileName,
+ RepoType: types.RepositoryType(repoType),
+ }
+ content, err = c.gs.GetRepoFileRaw(context.Background(), getFileRawReq)
+ if err != nil {
+ slog.Error("failed to get file content", slog.String("namespace", namespace),
+ slog.String("file", fileName), slog.String("repo", repoName), slog.String("ref", ref),
+ slog.Any("error", err))
+ return "", fmt.Errorf("failed to get file content,cause: %w", err)
+ }
+ slog.Debug("get file content success", slog.String("repoType", repoType), slog.String("namespace", namespace),
+ slog.String("file", fileName), slog.String("repo", repoName), slog.String("ref", ref))
+
+ return content, nil
+}
+
+// update repo relations
+func (c *GitCallbackComponent) updateRepoRelations(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool, fileNames []string) {
+ slog.Debug("update model relation for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("repoType", repoType), slog.Any("fileName", fileName), slog.Any("branch", ref))
+ if repoType == fmt.Sprintf("%ss", types.ModelRepo) {
+ c.updateModelRuntimeFrameworks(ctx, repoType, namespace, repoName, ref, fileName, deleteAction)
+ }
+ if repoType == fmt.Sprintf("%ss", types.DatasetRepo) {
+ c.updateDatasetTags(ctx, namespace, repoName, fileNames)
+ }
+}
+
+// update dataset tags for evaluation
+func (c *GitCallbackComponent) updateDatasetTags(ctx context.Context, namespace, repoName string, fileNames []string) {
+ // script dataset repo was not supported so far
+ scriptName := fmt.Sprintf("%s.py", repoName)
+ if slices.Contains(fileNames, scriptName) {
+ return
+ }
+ repo, err := c.rs.FindByPath(ctx, types.DatasetRepo, namespace, repoName)
+ if err != nil || repo == nil {
+ slog.Warn("fail to query repo for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err))
+ return
+ }
+ // check if it's evaluation dataset
+ evalDataset, err := c.dt.FindByRepo(ctx, string(types.EvaluationCategory), namespace, repoName, string(types.DatasetRepo))
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ // check if it's a mirror repo
+ mirror, err := c.mirrorStore.FindByRepoPath(ctx, types.DatasetRepo, namespace, repoName)
+ if err != nil || mirror == nil {
+ slog.Debug("fail to query mirror dataset for in callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err))
+ return
+ }
+ namespace := strings.Split(mirror.SourceRepoPath, "/")[0]
+ name := strings.Split(mirror.SourceRepoPath, "/")[1]
+ // use mirror namespace and name to find dataset
+ evalDataset, err = c.dt.FindByRepo(ctx, string(types.EvaluationCategory), namespace, name, string(types.DatasetRepo))
+ if err != nil {
+ slog.Debug("not an evaluation dataset, ignore it", slog.Any("repo id", repo.Path))
+ return
+ }
+ } else {
+ slog.Error("failed to query evaluation dataset", slog.Any("repo id", repo.Path), slog.Any("error", err))
+ return
+ }
+
+ }
+ tagIds := []int64{}
+ tagIds = append(tagIds, evalDataset.Tag.ID)
+ if evalDataset.RuntimeFramework != "" {
+ rTag, _ := c.ts.FindTag(ctx, evalDataset.RuntimeFramework, string(types.DatasetRepo), "runtime_framework")
+ if rTag != nil {
+ tagIds = append(tagIds, rTag.ID)
+ }
+ }
+
+ err = c.ts.UpsertRepoTags(ctx, repo.ID, []int64{}, tagIds)
+ if err != nil {
+ slog.Warn("fail to add dataset tag", slog.Any("repoId", repo.ID), slog.Any("tag id", tagIds), slog.Any("error", err))
+ }
+
+}
+
+// update model runtime frameworks
+func (c *GitCallbackComponent) updateModelRuntimeFrameworks(ctx context.Context, repoType, namespace, repoName, ref, fileName string, deleteAction bool) {
+ // must be model repo and config.json
+ if repoType != fmt.Sprintf("%ss", types.ModelRepo) || fileName != component.ConfigFileName || (ref != ("refs/heads/"+component.MainBranch) && ref != ("refs/heads/"+component.MasterBranch)) {
+ return
+ }
+ repo, err := c.rs.FindByPath(ctx, types.ModelRepo, namespace, repoName)
+ if err != nil || repo == nil {
+ slog.Warn("fail to query repo for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err))
+ return
+ }
+ // delete event
+ if deleteAction {
+ err := c.rrf.DeleteByRepoID(ctx, repo.ID)
+ if err != nil {
+ slog.Warn("fail to remove repo runtimes for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("repoid", repo.ID), slog.Any("error", err))
+ }
+ return
+ }
+ arch, err := c.rac.GetArchitectureFromConfig(ctx, namespace, repoName)
+ if err != nil {
+ slog.Warn("fail to get config.json content for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("error", err))
+ return
+ }
+ slog.Debug("get arch for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("arch", arch))
+ //add resource tag, like ascend
+ runtime_framework_tags, _ := c.ts.GetTagsByScopeAndCategories(ctx, "model", []string{"runtime_framework", "resource"})
+ fields := strings.Split(repo.Path, "/")
+ err = c.rac.AddResourceTag(ctx, runtime_framework_tags, fields[1], repo.ID)
+ if err != nil {
+ slog.Warn("fail to add resource tag", slog.Any("error", err))
+ return
+ }
+ runtimes, err := c.ras.ListByRArchNameAndModel(ctx, arch, fields[1])
+ // to do check resource models
+ if err != nil {
+ slog.Warn("fail to get runtime ids by arch for git callback", slog.Any("arch", arch), slog.Any("error", err))
+ return
+ }
+ slog.Debug("get runtimes by arch for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("arch", arch), slog.Any("runtimes", runtimes))
+ var frameIDs []int64
+ for _, runtime := range runtimes {
+ frameIDs = append(frameIDs, runtime.RuntimeFrameworkID)
+ }
+ slog.Debug("get new frame ids for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("frameIDs", frameIDs))
+ newFrames, err := c.rfs.ListByIDs(ctx, frameIDs)
+ if err != nil {
+ slog.Warn("fail to get runtime frameworks for git callback", slog.Any("arch", arch), slog.Any("error", err))
+ return
+ }
+ slog.Debug("get new frames by arch for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("newFrames", newFrames))
+ var newFrameMap = make(map[string]string)
+ for _, frame := range newFrames {
+ newFrameMap[strconv.FormatInt(frame.ID, 10)] = strconv.FormatInt(frame.ID, 10)
+ }
+ slog.Debug("get new frame map by arch for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("newFrameMap", newFrameMap))
+ oldRepoRuntimes, err := c.rrf.GetByRepoIDs(ctx, repo.ID)
+ if err != nil {
+ slog.Warn("fail to get repo runtimes for git callback", slog.Any("repo.ID", repo.ID), slog.Any("error", err))
+ return
+ }
+ slog.Debug("get old frames by arch for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("oldRepoRuntimes", oldRepoRuntimes))
+ var oldFrameMap = make(map[string]string)
+ // get map
+ for _, runtime := range oldRepoRuntimes {
+ oldFrameMap[strconv.FormatInt(runtime.RuntimeFrameworkID, 10)] = strconv.FormatInt(runtime.RuntimeFrameworkID, 10)
+ }
+ slog.Debug("get old frame map by arch for git callback", slog.Any("namespace", namespace), slog.Any("repoName", repoName), slog.Any("oldFrameMap", oldFrameMap))
+ // remove incorrect relation
+ for _, old := range oldRepoRuntimes {
+ // check if it need remove
+ _, exist := newFrameMap[strconv.FormatInt(old.RuntimeFrameworkID, 10)]
+ if !exist {
+ // remove incorrect relations
+ err := c.rrf.Delete(ctx, old.RuntimeFrameworkID, repo.ID, old.Type)
+ if err != nil {
+ slog.Warn("fail to delete old repo runtimes for git callback", slog.Any("repo.ID", repo.ID), slog.Any("runtime framework id", old.RuntimeFrameworkID), slog.Any("error", err))
+ }
+ // remove runtime framework tags
+ c.rac.RemoveRuntimeFrameworkTag(ctx, runtime_framework_tags, repo.ID, old.RuntimeFrameworkID)
+ }
+ }
+
+ // add new relation
+ for _, new := range newFrames {
+ // check if it need add
+ _, exist := oldFrameMap[strconv.FormatInt(new.ID, 10)]
+ if !exist {
+ // add new relations
+ err := c.rrf.Add(ctx, new.ID, repo.ID, new.Type)
+ if err != nil {
+ slog.Warn("fail to add new repo runtimes for git callback", slog.Any("repo.ID", repo.ID), slog.Any("runtime framework id", new.ID), slog.Any("error", err))
+ }
+ // add runtime framework and resource tags
+ err = c.rac.AddRuntimeFrameworkTag(ctx, runtime_framework_tags, repo.ID, new.ID)
+ if err != nil {
+ slog.Warn("fail to add runtime framework tag for git callback", slog.Any("repo.ID", repo.ID), slog.Any("runtime framework id", new.ID), slog.Any("error", err))
+ }
+ }
+ }
+
+}
+
+
+
package callback
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "slices"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component/tagparser"
+)
+
+type repoRelationWatcher struct {
+ ops []func() error
+ rs database.RepoStore
+ rrs database.RepoRelationsStore
+ gs gitserver.GitServer
+
+ readmeStatus string
+}
+
+func WatchRepoRelation(req *types.GiteaCallbackPushReq, ss database.RepoStore,
+ rrs database.RepoRelationsStore,
+ gs gitserver.GitServer) Watcher {
+ watcher := new(repoRelationWatcher)
+ watcher.rs = ss
+ watcher.rrs = rrs
+ watcher.gs = gs
+
+ //only care about main branch
+ if req.Ref != "refs/heads/main" {
+ return watcher
+ }
+ // split req.Repository.FullName by '/'
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+
+ if repoType == fmt.Sprintf("%ss", types.CodeRepo) {
+ return watcher
+ }
+
+ commits := req.Commits
+ ref := req.Ref
+ for _, commit := range commits {
+ if slices.Contains(commit.Modified, types.ReadmeFileName) {
+ watcher.readmeStatus = "modified"
+ continue
+ }
+ if slices.Contains(commit.Added, types.ReadmeFileName) {
+ if watcher.readmeStatus != "modified" {
+ watcher.readmeStatus = "added"
+ }
+ continue
+ }
+ if slices.Contains(commit.Removed, types.ReadmeFileName) {
+ watcher.readmeStatus = "removed"
+ continue
+ }
+ }
+
+ //readme file not changed in this whole push, so do nothing
+ if watcher.readmeStatus == "" {
+ return watcher
+ }
+
+ watcher.regenerate(namespace, repoName, repoType, ref)
+
+ return watcher
+}
+
+func (w *repoRelationWatcher) Run() error {
+ var err error
+ for _, op := range w.ops {
+ err = errors.Join(err, op())
+ }
+ return err
+}
+
+func (w *repoRelationWatcher) toRepoIDsFromReadme(namespace, repoName, repoType, ref string) ([]int64, error) {
+ var readme string
+ var err error
+ var toRepoIDs []int64
+ var paths []string
+
+ readme, err = w.getFileRaw(repoType, namespace, repoName, ref, types.ReadmeFileName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get readme file,%w", err)
+ }
+ meta, err := tagparser.MetaTags(readme)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse readme file meta,%w", err)
+ }
+ if repoType == fmt.Sprintf("%ss", types.ModelRepo) {
+ datasetItems := meta["datasets"]
+ codeItems := meta["codes"]
+ for _, datasetItem := range datasetItems {
+ paths = append(paths, fmt.Sprintf("%s%s", "datasets_", datasetItem))
+ }
+ for _, codeItem := range codeItems {
+ paths = append(paths, fmt.Sprintf("%s%s", "codes_", codeItem))
+ }
+ }
+
+ if repoType == fmt.Sprintf("%ss", types.SpaceRepo) || repoType == fmt.Sprintf("%ss", types.DatasetRepo) || repoType == fmt.Sprintf("%ss", types.PromptRepo) {
+ modelItems := meta["models"]
+ if len(modelItems) == 0 {
+ return toRepoIDs, nil
+ }
+ for _, modelItem := range modelItems {
+ paths = append(paths, fmt.Sprintf("%s%s", "models_", modelItem))
+ }
+ }
+ slog.Info("git call back update relations", slog.Any("from_repo_type", repoType), slog.String("from_repo", fmt.Sprintf("%s/%s", namespace, repoName)), slog.Any("to_repos", paths))
+ if len(paths) > 0 {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ toRepos, err := w.rs.FindByGitPaths(ctx, paths, database.Columns("id"))
+ cancel()
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repos by paths,%w", err)
+ }
+ for _, repo := range toRepos {
+ toRepoIDs = append(toRepoIDs, repo.ID)
+ }
+
+ }
+ return toRepoIDs, nil
+}
+func (w *repoRelationWatcher) regenerate(namespace, repoName, repoType, ref string) *repoRelationWatcher {
+ w.ops = append(w.ops,
+ func() error {
+ var fromRepoID int64
+ var err error
+ var toRepoIDs []int64
+
+ if w.readmeStatus != "removed" {
+ toRepoIDs, err = w.toRepoIDsFromReadme(namespace, repoName, repoType, ref)
+ if err != nil {
+ return fmt.Errorf("failed to get relation to repos from readme,%w", err)
+ }
+ }
+ //TODO: get to repo ids from app.py by parsing model ids
+
+ ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+ defer cancel()
+
+ if repoType == fmt.Sprintf("%ss", types.ModelRepo) {
+ fromRepo, err := w.rs.FindByPath(ctx, types.ModelRepo, namespace, repoName)
+ if err != nil {
+ return fmt.Errorf("failed to find model repo,%w", err)
+ }
+ fromRepoID = fromRepo.ID
+ }
+
+ if repoType == fmt.Sprintf("%ss", types.SpaceRepo) {
+ fromRepo, err := w.rs.FindByPath(ctx, types.SpaceRepo, namespace, repoName)
+ if err != nil {
+ return fmt.Errorf("failed to find space repo,%w", err)
+ }
+ fromRepoID = fromRepo.ID
+ }
+
+ if repoType == fmt.Sprintf("%ss", types.DatasetRepo) {
+ fromRepo, err := w.rs.FindByPath(ctx, types.DatasetRepo, namespace, repoName)
+ if err != nil {
+ return fmt.Errorf("failed to find dataset repo,%w", err)
+ }
+ fromRepoID = fromRepo.ID
+ }
+
+ if repoType == fmt.Sprintf("%ss", types.PromptRepo) {
+ fromRepo, err := w.rs.FindByPath(ctx, types.PromptRepo, namespace, repoName)
+ if err != nil {
+ return fmt.Errorf("failed to find prompt repo,%w", err)
+ }
+ fromRepoID = fromRepo.ID
+ }
+
+ return w.rrs.Override(ctx, fromRepoID, toRepoIDs...)
+ })
+
+ return w
+}
+
+func (w *repoRelationWatcher) getFileRaw(repoType, namespace, repoName, ref, fileName string) (string, error) {
+ var (
+ content string
+ err error
+ )
+ repoType = strings.TrimRight(repoType, "s")
+ getFileRawReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: repoName,
+ Ref: ref,
+ Path: fileName,
+ RepoType: types.RepositoryType(repoType),
+ }
+ content, err = w.gs.GetRepoFileRaw(context.Background(), getFileRawReq)
+ if err != nil {
+ slog.Error("failed to get file content", slog.String("namespace", namespace),
+ slog.String("file", fileName), slog.String("repo", repoName), slog.String("ref", ref),
+ slog.Any("error", err))
+ return "", fmt.Errorf("failed to get file content,cause: %w", err)
+ }
+ slog.Debug("get file content success", slog.String("repoType", repoType), slog.String("namespace", namespace),
+ slog.String("file", fileName), slog.String("repo", repoName), slog.String("ref", ref))
+
+ return content, nil
+}
+
+
+
package callback
+
+import (
+ "errors"
+ "slices"
+ "strings"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+type repoTagWatcher struct {
+ ops []func() error
+}
+
+func WatchRepoTag(req *types.GiteaCallbackPushReq) Watcher {
+ watcher := new(repoTagWatcher)
+
+ commits := req.Commits
+ ref := req.Ref
+ // split req.Repository.FullName by '/'
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+ for _, commit := range commits {
+ if slices.Contains(commit.Modified, types.ReadmeFileName) {
+ watcher.modify(namespace, repoName, repoType, ref)
+ continue
+ }
+ if slices.Contains(commit.Added, types.ReadmeFileName) {
+ watcher.add(namespace, repoName, repoType, ref)
+ continue
+ }
+ if slices.Contains(commit.Removed, types.ReadmeFileName) {
+ watcher.del(namespace, repoName, repoType, ref)
+ continue
+ }
+ }
+
+ return watcher
+}
+
+func (w *repoTagWatcher) Run() error {
+ var err error
+ for _, op := range w.ops {
+ err = errors.Join(op())
+ }
+ return err
+}
+
+func (w *repoTagWatcher) modify(namespace, name, repoType, ref string) *repoTagWatcher {
+ w.ops = append(w.ops,
+ func() error {
+ return nil
+ })
+ return w
+}
+
+func (w *repoTagWatcher) add(namespace, name, repoType, ref string) *repoTagWatcher {
+ w.ops = append(w.ops,
+ func() error {
+ return nil
+ })
+ return w
+}
+
+func (w *repoTagWatcher) del(namespace, name, repoType, ref string) *repoTagWatcher {
+ w.ops = append(w.ops,
+ func() error {
+ return nil
+ })
+ return w
+}
+
+
+
package callback
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component"
+)
+
+type spaceDeployWatcher struct {
+ ops []func() error
+ ss database.SpaceStore
+ sc component.SpaceComponent
+}
+
+func WatchSpaceChange(req *types.GiteaCallbackPushReq, ss database.SpaceStore, sc component.SpaceComponent) Watcher {
+ watcher := new(spaceDeployWatcher)
+ watcher.ss = ss
+ watcher.sc = sc
+ // split req.Repository.FullName by '/' for example: <repotype>_<namespace>/<reponame>
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+
+ if repoType != "spaces" {
+ return watcher
+ }
+ // username = namespace in fullname of gitea
+ watcher.deploy(namespace, repoName, namespace)
+ return watcher
+}
+
+func (w *spaceDeployWatcher) Run() error {
+ var err error
+ for _, op := range w.ops {
+ err = errors.Join(err, op())
+ }
+ return err
+}
+
+func (w *spaceDeployWatcher) deploy(namespace string, repoName string, currentUser string) *spaceDeployWatcher {
+ w.ops = append(w.ops,
+ func() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
+ defer cancel()
+
+ space, err := w.ss.FindByPath(ctx, namespace, repoName)
+ if err != nil {
+ return fmt.Errorf("failed to find space: %w", err)
+ }
+
+ w.sc.FixHasEntryFile(ctx, space)
+ if !space.HasAppFile {
+ return nil
+ }
+ // trigger space deployment by gitea call back
+ _, err = w.sc.Deploy(ctx, namespace, repoName, currentUser)
+ if err != nil {
+ return fmt.Errorf("failed to trigger space delopy: %w", err)
+ } else {
+ return nil
+ }
+ })
+
+ return w
+}
+
+
+
package callback
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type SyncVersionGenerator struct {
+ s database.MultiSyncStore
+}
+
+func NewSyncVersionGenerator() *SyncVersionGenerator {
+ return &SyncVersionGenerator{
+ s: database.NewMultiSyncStore(),
+ }
+}
+
+func (g *SyncVersionGenerator) GenSyncVersion(req *types.GiteaCallbackPushReq) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
+ defer cancel()
+ splits := strings.Split(req.Repository.FullName, "/")
+ fullNamespace, repoName := splits[0], splits[1]
+ repoType, namespace, _ := strings.Cut(fullNamespace, "_")
+ _, err := g.s.Create(ctx, database.SyncVersion{
+ SourceID: types.SyncVersionSourceOpenCSG,
+ RepoPath: fmt.Sprintf("%s/%s", namespace, repoName),
+ RepoType: types.RepositoryType(strings.TrimRight(repoType, "s")),
+ LastModifiedAt: req.HeadCommit.LastModifyTime,
+ ChangeLog: req.HeadCommit.Message,
+ })
+
+ return err
+}
+
+
+
package component
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type ClusterComponent interface {
+ Index(ctx context.Context) ([]types.ClusterRes, error)
+ GetClusterById(ctx context.Context, clusterId string) (*types.ClusterRes, error)
+ Update(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error)
+}
+
+func NewClusterComponent(config *config.Config) (ClusterComponent, error) {
+ c := &clusterComponentImpl{}
+ c.deployer = deploy.NewDeployer()
+
+ return c, nil
+}
+
+type clusterComponentImpl struct {
+ deployer deploy.Deployer
+}
+
+func (c *clusterComponentImpl) Index(ctx context.Context) ([]types.ClusterRes, error) {
+ return c.deployer.ListCluster(ctx)
+}
+
+func (c *clusterComponentImpl) GetClusterById(ctx context.Context, clusterId string) (*types.ClusterRes, error) {
+ return c.deployer.GetClusterById(ctx, clusterId)
+}
+
+func (c *clusterComponentImpl) Update(ctx context.Context, data types.ClusterRequest) (*types.UpdateClusterResponse, error) {
+ return c.deployer.UpdateCluster(ctx, data)
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+const codeGitattributesContent = modelGitattributesContent
+
+type CodeComponent interface {
+ Create(ctx context.Context, req *types.CreateCodeReq) (*types.Code, error)
+ Index(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Code, int, error)
+ Update(ctx context.Context, req *types.UpdateCodeReq) (*types.Code, error)
+ Delete(ctx context.Context, namespace, name, currentUser string) error
+ Show(ctx context.Context, namespace, name, currentUser string) (*types.Code, error)
+ Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error)
+ OrgCodes(ctx context.Context, req *types.OrgCodesReq) ([]types.Code, int, error)
+}
+
+func NewCodeComponent(config *config.Config) (CodeComponent, error) {
+ c := &codeComponentImpl{}
+ var err error
+ c.repoComponent, err = NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, err
+ }
+ c.codeStore = database.NewCodeStore()
+ c.repoStore = database.NewRepoStore()
+ gs, err := git.NewGitServer(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server, error: %w", err)
+ }
+ c.gitServer = gs
+ c.config = config
+ c.userLikesStore = database.NewUserLikesStore()
+ c.userSvcClient = rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+ return c, nil
+}
+
+type codeComponentImpl struct {
+ config *config.Config
+ repoComponent RepoComponent
+ codeStore database.CodeStore
+ repoStore database.RepoStore
+ userLikesStore database.UserLikesStore
+ gitServer gitserver.GitServer
+ userSvcClient rpc.UserSvcClient
+}
+
+func (c *codeComponentImpl) Create(ctx context.Context, req *types.CreateCodeReq) (*types.Code, error) {
+ var (
+ nickname string
+ tags []types.RepoTag
+ )
+
+ if req.Nickname != "" {
+ nickname = req.Nickname
+ } else {
+ nickname = req.Name
+ }
+
+ if req.DefaultBranch == "" {
+ req.DefaultBranch = types.MainBranch
+ }
+
+ req.RepoType = types.CodeRepo
+ req.Readme = generateReadmeData(req.License)
+ req.Nickname = nickname
+ _, dbRepo, err := c.repoComponent.CreateRepo(ctx, req.CreateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ dbCode := database.Code{
+ Repository: dbRepo,
+ RepositoryID: dbRepo.ID,
+ }
+
+ code, err := c.codeStore.Create(ctx, dbCode)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create database code, cause: %w", err)
+ }
+
+ // Create README.md file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: dbRepo.User.Username,
+ Email: dbRepo.User.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: req.Readme,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: readmeFileName,
+ }, types.CodeRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create README.md file, cause: %w", err)
+ }
+
+ // Create .gitattributes file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: dbRepo.User.Username,
+ Email: dbRepo.User.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: codeGitattributesContent,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: gitattributesFileName,
+ }, types.CodeRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create .gitattributes file, cause: %w", err)
+ }
+
+ for _, tag := range code.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ resCode := &types.Code{
+ ID: code.ID,
+ Name: code.Repository.Name,
+ Nickname: code.Repository.Nickname,
+ Description: code.Repository.Description,
+ Likes: code.Repository.Likes,
+ Downloads: code.Repository.DownloadCount,
+ Path: code.Repository.Path,
+ RepositoryID: code.RepositoryID,
+ Repository: common.BuildCloneInfo(c.config, code.Repository),
+ Private: code.Repository.Private,
+ User: types.User{
+ Username: dbRepo.User.Username,
+ Nickname: dbRepo.User.NickName,
+ Email: dbRepo.User.Email,
+ },
+ Tags: tags,
+ CreatedAt: code.CreatedAt,
+ UpdatedAt: code.UpdatedAt,
+ }
+
+ return resCode, nil
+}
+
+func (c *codeComponentImpl) Index(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Code, int, error) {
+ var (
+ err error
+ resCodes []types.Code
+ )
+ repos, total, err := c.repoComponent.PublicToUser(ctx, types.CodeRepo, filter.Username, filter, per, page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public code repos,error:%w", err)
+ return nil, 0, newError
+ }
+ var repoIDs []int64
+ for _, repo := range repos {
+ repoIDs = append(repoIDs, repo.ID)
+ }
+ codes, err := c.codeStore.ByRepoIDs(ctx, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get codes by repo ids,error:%w", err)
+ return nil, 0, newError
+ }
+
+ //loop through repos to keep the repos in sort order
+ for _, repo := range repos {
+ var code *database.Code
+ for _, c := range codes {
+ if c.RepositoryID == repo.ID {
+ code = &c
+ code.Repository = repo
+ break
+ }
+ }
+ var tags []types.RepoTag
+ for _, tag := range repo.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ resCodes = append(resCodes, types.Code{
+ ID: code.ID,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ Likes: repo.Likes,
+ Downloads: repo.DownloadCount,
+ Path: repo.Path,
+ RepositoryID: repo.ID,
+ Private: repo.Private,
+ CreatedAt: code.CreatedAt,
+ UpdatedAt: repo.UpdatedAt,
+ Tags: tags,
+ Source: repo.Source,
+ SyncStatus: repo.SyncStatus,
+ License: repo.License,
+ })
+ }
+
+ return resCodes, total, nil
+}
+
+func (c *codeComponentImpl) Update(ctx context.Context, req *types.UpdateCodeReq) (*types.Code, error) {
+ req.RepoType = types.CodeRepo
+ dbRepo, err := c.repoComponent.UpdateRepo(ctx, req.UpdateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ code, err := c.codeStore.ByRepoID(ctx, dbRepo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find code repo, error: %w", err)
+ }
+
+ //update times of code
+ err = c.codeStore.Update(ctx, *code)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update database code repo, error: %w", err)
+ }
+
+ resCode := &types.Code{
+ ID: code.ID,
+ Name: dbRepo.Name,
+ Nickname: dbRepo.Nickname,
+ Description: dbRepo.Description,
+ Likes: dbRepo.Likes,
+ Downloads: dbRepo.DownloadCount,
+ Path: dbRepo.Path,
+ RepositoryID: dbRepo.ID,
+ Private: dbRepo.Private,
+ CreatedAt: code.CreatedAt,
+ UpdatedAt: code.UpdatedAt,
+ }
+
+ return resCode, nil
+}
+
+func (c *codeComponentImpl) Delete(ctx context.Context, namespace, name, currentUser string) error {
+ code, err := c.codeStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find code, error: %w", err)
+ }
+
+ deleteDatabaseRepoReq := types.DeleteRepoReq{
+ Username: currentUser,
+ Namespace: namespace,
+ Name: name,
+ RepoType: types.CodeRepo,
+ }
+ _, err = c.repoComponent.DeleteRepo(ctx, deleteDatabaseRepoReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete repo of code, error: %w", err)
+ }
+
+ err = c.codeStore.Delete(ctx, *code)
+ if err != nil {
+ return fmt.Errorf("failed to delete database code, error: %w", err)
+ }
+ return nil
+}
+
+func (c *codeComponentImpl) Show(ctx context.Context, namespace, name, currentUser string) (*types.Code, error) {
+ var tags []types.RepoTag
+ code, err := c.codeStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find code, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, currentUser, code.Repository)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ ns, err := c.repoComponent.GetNameSpaceInfo(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace info for code, error: %w", err)
+ }
+
+ for _, tag := range code.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ likeExists, err := c.userLikesStore.IsExist(ctx, currentUser, code.Repository.ID)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user likes,error:%w", err)
+ return nil, newError
+ }
+
+ resCode := &types.Code{
+ ID: code.ID,
+ Name: code.Repository.Name,
+ Nickname: code.Repository.Nickname,
+ Description: code.Repository.Description,
+ Likes: code.Repository.Likes,
+ Downloads: code.Repository.DownloadCount,
+ Path: code.Repository.Path,
+ RepositoryID: code.Repository.ID,
+ DefaultBranch: code.Repository.DefaultBranch,
+ Repository: common.BuildCloneInfo(c.config, code.Repository),
+ Tags: tags,
+ User: types.User{
+ Username: code.Repository.User.Username,
+ Nickname: code.Repository.User.NickName,
+ Email: code.Repository.User.Email,
+ },
+ Private: code.Repository.Private,
+ CreatedAt: code.CreatedAt,
+ UpdatedAt: code.Repository.UpdatedAt,
+ UserLikes: likeExists,
+ Source: code.Repository.Source,
+ SyncStatus: code.Repository.SyncStatus,
+ License: code.Repository.License,
+ CanWrite: permission.CanWrite,
+ CanManage: permission.CanAdmin,
+ Namespace: ns,
+ }
+ if permission.CanAdmin {
+ resCode.SensitiveCheckStatus = code.Repository.SensitiveCheckStatus.String()
+ }
+
+ return resCode, nil
+}
+
+func (c *codeComponentImpl) Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error) {
+ code, err := c.codeStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find code repo, error: %w", err)
+ }
+
+ allow, _ := c.repoComponent.AllowReadAccessRepo(ctx, code.Repository, currentUser)
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ return c.getRelations(ctx, code.RepositoryID, currentUser)
+}
+
+func (c *codeComponentImpl) getRelations(ctx context.Context, repoID int64, currentUser string) (*types.Relations, error) {
+ res, err := c.repoComponent.RelatedRepos(ctx, repoID, currentUser)
+ if err != nil {
+ return nil, err
+ }
+ rels := new(types.Relations)
+ modelRepos := res[types.ModelRepo]
+ for _, repo := range modelRepos {
+ rels.Models = append(rels.Models, &types.Model{
+ Path: repo.Path,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ UpdatedAt: repo.UpdatedAt,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ })
+ }
+
+ return rels, nil
+}
+
+func (c *codeComponentImpl) OrgCodes(ctx context.Context, req *types.OrgCodesReq) ([]types.Code, int, error) {
+ var resCodes []types.Code
+ var err error
+ r := membership.RoleUnknown
+ if req.CurrentUser != "" {
+ r, err = c.userSvcClient.GetMemberRole(ctx, req.Namespace, req.CurrentUser)
+ // log error, and treat user as unknown role in org
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Namespace), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ }
+ onlyPublic := !r.CanRead()
+ codes, total, err := c.codeStore.ByOrgPath(ctx, req.Namespace, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get org codes,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range codes {
+ resCodes = append(resCodes, types.Code{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resCodes, total, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type CollectionComponent interface {
+ GetCollections(ctx context.Context, filter *types.CollectionFilter, per, page int) ([]types.Collection, int, error)
+ CreateCollection(ctx context.Context, input types.CreateCollectionReq) (*database.Collection, error)
+ GetCollection(ctx context.Context, currentUser string, id int64) (*types.Collection, error)
+ // get non private repositories of the collection
+ GetPublicRepos(collection types.Collection) []types.CollectionRepository
+ UpdateCollection(ctx context.Context, input types.CreateCollectionReq) (*database.Collection, error)
+ DeleteCollection(ctx context.Context, id int64, userName string) error
+ AddReposToCollection(ctx context.Context, req types.UpdateCollectionReposReq) error
+ RemoveReposFromCollection(ctx context.Context, req types.UpdateCollectionReposReq) error
+ OrgCollections(ctx context.Context, req *types.OrgCollectionsReq) ([]types.Collection, int, error)
+}
+
+func NewCollectionComponent(config *config.Config) (CollectionComponent, error) {
+ cc := &collectionComponentImpl{}
+ cc.collectionStore = database.NewCollectionStore()
+ cc.repoStore = database.NewRepoStore()
+ cc.userStore = database.NewUserStore()
+ cc.orgStore = database.NewOrgStore()
+ cc.userLikesStore = database.NewUserLikesStore()
+ cc.userSvcClient = rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+ spaceComponent, err := NewSpaceComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ cc.spaceComponent = spaceComponent
+ return cc, nil
+}
+
+type collectionComponentImpl struct {
+ collectionStore database.CollectionStore
+ orgStore database.OrgStore
+ repoStore database.RepoStore
+ userStore database.UserStore
+ userLikesStore database.UserLikesStore
+ userSvcClient rpc.UserSvcClient
+ spaceComponent SpaceComponent
+}
+
+func (cc *collectionComponentImpl) GetCollections(ctx context.Context, filter *types.CollectionFilter, per, page int) ([]types.Collection, int, error) {
+ collections, total, err := cc.collectionStore.GetCollections(ctx, filter, per, page, true)
+ if err != nil {
+ return nil, 0, err
+ }
+ var newCollection []types.Collection
+ temporaryVariable, _ := json.Marshal(collections)
+ err = json.Unmarshal(temporaryVariable, &newCollection)
+ if err != nil {
+ return nil, 0, err
+ }
+ return newCollection, total, nil
+
+}
+
+func (cc *collectionComponentImpl) CreateCollection(ctx context.Context, input types.CreateCollectionReq) (*database.Collection, error) {
+ // find by user name
+ user, err := cc.userStore.FindByUsername(ctx, input.Username)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find user for collection, %w", err)
+ }
+ collection := database.Collection{
+ Username: user.Username,
+ Namespace: input.Namespace,
+ UserID: user.ID,
+ Name: input.Name,
+ Nickname: input.Nickname,
+ Description: input.Description,
+ Private: input.Private,
+ Theme: input.Theme,
+ }
+ //for org case, no need user name
+ if input.Namespace != "" {
+ collection.Username = ""
+ }
+
+ return cc.collectionStore.CreateCollection(ctx, collection)
+}
+
+func (cc *collectionComponentImpl) GetCollection(ctx context.Context, currentUser string, id int64) (*types.Collection, error) {
+ collection, err := cc.collectionStore.GetCollection(ctx, id)
+ if err != nil {
+ return nil, err
+ }
+ // find by user name
+ avatar := ""
+ if collection.Username != "" {
+ user, err := cc.userStore.FindByUsername(ctx, collection.Username)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find user for collection, %w", err)
+ }
+ avatar = user.Avatar
+ } else if collection.Namespace != "" {
+ org, err := cc.orgStore.FindByPath(ctx, collection.Namespace)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get org info, path: %s, error: %w", collection.Namespace, err)
+ }
+ avatar = org.Logo
+ }
+
+ permission, err := cc.getUserCollectionPermission(ctx, currentUser, collection)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ var newCollection types.Collection
+ temporaryVariable, _ := json.Marshal(collection)
+ err = json.Unmarshal(temporaryVariable, &newCollection)
+ if err != nil {
+ return nil, err
+ }
+ likeExists, err := cc.userLikesStore.IsExistCollection(ctx, currentUser, id)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user likes,error:%w", err)
+ return nil, newError
+ }
+ if !permission.CanWrite {
+ newCollection.Repositories = cc.GetPublicRepos(newCollection)
+ }
+ for i, repo := range newCollection.Repositories {
+ if repo.RepositoryType == types.SpaceRepo && strings.Contains(repo.Path, "/") {
+ namespace, name := repo.NamespaceAndName()
+ _, status, _ := cc.spaceComponent.Status(ctx, namespace, name)
+ newCollection.Repositories[i].Status = status
+ }
+ }
+ newCollection.UserLikes = likeExists
+ newCollection.CanWrite = permission.CanWrite
+ newCollection.CanManage = permission.CanAdmin
+ newCollection.Avatar = avatar
+ return &newCollection, nil
+}
+
+// get non private repositories of the collection
+func (cc *collectionComponentImpl) GetPublicRepos(collection types.Collection) []types.CollectionRepository {
+ var filtered []types.CollectionRepository
+ for _, repo := range collection.Repositories {
+ if !repo.Private {
+ filtered = append(filtered, repo)
+ }
+ }
+ return filtered
+}
+
+func (cc *collectionComponentImpl) UpdateCollection(ctx context.Context, input types.CreateCollectionReq) (*database.Collection, error) {
+ collection, err := cc.collectionStore.GetCollection(ctx, input.ID)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find collection to update, %w", err)
+ }
+ collection.Name = input.Name
+ collection.Nickname = input.Nickname
+ collection.Description = input.Description
+ collection.Private = input.Private
+ collection.Theme = input.Theme
+ collection.UpdatedAt = time.Now()
+ return cc.collectionStore.UpdateCollection(ctx, *collection)
+}
+
+func (cc *collectionComponentImpl) DeleteCollection(ctx context.Context, id int64, userName string) error {
+ // find by user name
+ user, err := cc.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return fmt.Errorf("cannot find user for collection, %w", err)
+ }
+ return cc.collectionStore.DeleteCollection(ctx, id, user.ID)
+}
+
+func (cc *collectionComponentImpl) AddReposToCollection(ctx context.Context, req types.UpdateCollectionReposReq) error {
+ // find by user name
+ user, err := cc.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return fmt.Errorf("cannot find user for collection, %w", err)
+ }
+ collection, err := cc.collectionStore.GetCollection(ctx, req.ID)
+ if err != nil {
+ return err
+ }
+ if collection.UserID != user.ID {
+ return fmt.Errorf("no permission to operate this collection: %s", strconv.FormatInt(req.ID, 10))
+ }
+ var collectionRepos []database.CollectionRepository
+ for _, id := range req.RepoIDs {
+ collectionRepos = append(collectionRepos, database.CollectionRepository{
+ CollectionID: req.ID,
+ RepositoryID: id,
+ })
+ }
+ return cc.collectionStore.AddCollectionRepos(ctx, collectionRepos)
+}
+
+func (cc *collectionComponentImpl) RemoveReposFromCollection(ctx context.Context, req types.UpdateCollectionReposReq) error {
+ // find by user name
+ user, err := cc.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return fmt.Errorf("cannot find user for collection, %w", err)
+ }
+ collection, err := cc.collectionStore.GetCollection(ctx, req.ID)
+ if err != nil {
+ return err
+ }
+ if collection.UserID != user.ID {
+ return fmt.Errorf("no permission to operate this collection: %s", strconv.FormatInt(req.ID, 10))
+ }
+ var collectionRepos []database.CollectionRepository
+ for _, id := range req.RepoIDs {
+ collectionRepos = append(collectionRepos, database.CollectionRepository{
+ CollectionID: req.ID,
+ RepositoryID: id,
+ })
+ }
+ return cc.collectionStore.RemoveCollectionRepos(ctx, collectionRepos)
+}
+
+func (cc *collectionComponentImpl) getUserCollectionPermission(ctx context.Context, userName string, collection *database.Collection) (*types.UserRepoPermission, error) {
+ if userName == "" {
+ //anonymous user only has read permission to public repo
+ return &types.UserRepoPermission{CanRead: !collection.Private, CanWrite: false, CanAdmin: false}, nil
+ }
+
+ namespace := collection.Namespace
+ namespaceType := "user"
+ if namespace == "" {
+ //Compatibility old data
+ namespace = collection.Username
+ }
+ if collection.Username != namespace {
+ namespaceType = "org"
+ }
+
+ if namespaceType == "user" {
+ //owner has full permission
+ if userName == namespace {
+ return &types.UserRepoPermission{
+ CanRead: true,
+ CanWrite: true,
+ CanAdmin: true,
+ }, nil
+ } else {
+ //other user has read permission to pubic repo
+ return &types.UserRepoPermission{
+ CanRead: !collection.Private, CanWrite: false, CanAdmin: false,
+ }, nil
+ }
+ } else {
+ r, err := cc.userSvcClient.GetMemberRole(ctx, namespace, userName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user '%s' member role of org '%s' when get user repo permission, error: %w", userName, namespace, err)
+ }
+
+ return &types.UserRepoPermission{
+ CanRead: r.CanRead() || !collection.Private,
+ CanWrite: r.CanWrite(),
+ CanAdmin: r.CanAdmin(),
+ }, nil
+ }
+}
+
+func (c *collectionComponentImpl) OrgCollections(ctx context.Context, req *types.OrgCollectionsReq) ([]types.Collection, int, error) {
+ var err error
+ r := membership.RoleUnknown
+ if req.CurrentUser != "" {
+ r, err = c.userSvcClient.GetMemberRole(ctx, req.Namespace, req.CurrentUser)
+ // log error, and treat user as unknown role in org
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Namespace), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ }
+ onlyPublic := !r.CanRead()
+ collections, total, err := c.collectionStore.ByUserOrgs(ctx, req.Namespace, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ return nil, 0, err
+ }
+ var newCollection []types.Collection
+ temporaryVariable, _ := json.Marshal(collections)
+ err = json.Unmarshal(temporaryVariable, &newCollection)
+ if err != nil {
+ return nil, 0, err
+ }
+ return newCollection, total, nil
+
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+const datasetGitattributesContent = `*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.lz4 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Audio files - uncompressed
+*.pcm filter=lfs diff=lfs merge=lfs -text
+*.sam filter=lfs diff=lfs merge=lfs -text
+*.raw filter=lfs diff=lfs merge=lfs -text
+# Audio files - compressed
+*.aac filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.mp3 filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+# Image files - uncompressed
+*.bmp filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.tiff filter=lfs diff=lfs merge=lfs -text
+# Image files - compressed
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+
+`
+
+const (
+ initCommitMessage = "initial commit"
+ ossFileExpire = 259200 * time.Second
+ readmeFileName = "README.md"
+ gitattributesFileName = ".gitattributes"
+)
+
+type DatasetComponent interface {
+ Create(ctx context.Context, req *types.CreateDatasetReq) (*types.Dataset, error)
+ Index(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Dataset, int, error)
+ Update(ctx context.Context, req *types.UpdateDatasetReq) (*types.Dataset, error)
+ Delete(ctx context.Context, namespace, name, currentUser string) error
+ Show(ctx context.Context, namespace, name, currentUser string) (*types.Dataset, error)
+ Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error)
+ OrgDatasets(ctx context.Context, req *types.OrgDatasetsReq) ([]types.Dataset, int, error)
+}
+
+func NewDatasetComponent(config *config.Config) (DatasetComponent, error) {
+ c := &datasetComponentImpl{}
+ c.tagStore = database.NewTagStore()
+ c.datasetStore = database.NewDatasetStore()
+ c.repoStore = database.NewRepoStore()
+ c.namespaceStore = database.NewNamespaceStore()
+ c.userStore = database.NewUserStore()
+ c.userLikesStore = database.NewUserLikesStore()
+ var err error
+ c.repoComponent, err = NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create repo component, error: %w", err)
+ }
+ c.sensitiveComponent, err = NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create sensitive component, error: %w", err)
+ }
+ gs, err := git.NewGitServer(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server, error: %w", err)
+ }
+ c.userSvcClient = rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+ c.gitServer = gs
+ c.config = config
+ return c, nil
+}
+
+type datasetComponentImpl struct {
+ config *config.Config
+ repoComponent RepoComponent
+ tagStore database.TagStore
+ datasetStore database.DatasetStore
+ repoStore database.RepoStore
+ namespaceStore database.NamespaceStore
+ userStore database.UserStore
+ sensitiveComponent SensitiveComponent
+ gitServer gitserver.GitServer
+ userLikesStore database.UserLikesStore
+ userSvcClient rpc.UserSvcClient
+}
+
+func (c *datasetComponentImpl) Create(ctx context.Context, req *types.CreateDatasetReq) (*types.Dataset, error) {
+ var (
+ nickname string
+ tags []types.RepoTag
+ )
+
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, errors.New("namespace does not exist")
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ if namespace.NamespaceType == database.OrgNamespace {
+ canWrite, err := c.repoComponent.CheckCurrentUserPermission(ctx, req.Username, req.Namespace, membership.RoleWrite)
+ if err != nil {
+ return nil, err
+ }
+ if !canWrite {
+ return nil, errors.New("users do not have permission to create datasets in this organization")
+ }
+ } else {
+ if namespace.Path != user.Username {
+ return nil, errors.New("users do not have permission to create datasets in this namespace")
+ }
+ }
+ }
+
+ if req.Nickname != "" {
+ nickname = req.Nickname
+ } else {
+ nickname = req.Name
+ }
+
+ if req.DefaultBranch == "" {
+ req.DefaultBranch = "main"
+ }
+
+ req.RepoType = types.DatasetRepo
+ req.Readme = generateReadmeData(req.License)
+ req.Nickname = nickname
+ _, dbRepo, err := c.repoComponent.CreateRepo(ctx, req.CreateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ dbDataset := database.Dataset{
+ Repository: dbRepo,
+ RepositoryID: dbRepo.ID,
+ }
+
+ dataset, err := c.datasetStore.Create(ctx, dbDataset)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create database dataset, cause: %w", err)
+ }
+
+ // Create README.md file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: user.Username,
+ Email: user.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: req.Readme,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: readmeFileName,
+ }, types.DatasetRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create README.md file, cause: %w", err)
+ }
+
+ // Create .gitattributes file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: user.Username,
+ Email: user.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: datasetGitattributesContent,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: gitattributesFileName,
+ }, types.DatasetRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create .gitattributes file, cause: %w", err)
+ }
+
+ for _, tag := range dataset.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ resDataset := &types.Dataset{
+ ID: dataset.ID,
+ Name: dataset.Repository.Name,
+ Nickname: dataset.Repository.Nickname,
+ Description: dataset.Repository.Description,
+ Likes: dataset.Repository.Likes,
+ Downloads: dataset.Repository.DownloadCount,
+ Path: dataset.Repository.Path,
+ RepositoryID: dataset.RepositoryID,
+ Repository: common.BuildCloneInfo(c.config, dataset.Repository),
+ Private: dataset.Repository.Private,
+ User: types.User{
+ Username: user.Username,
+ Nickname: user.NickName,
+ Email: user.Email,
+ },
+ Tags: tags,
+ CreatedAt: dataset.CreatedAt,
+ UpdatedAt: dataset.UpdatedAt,
+ }
+
+ return resDataset, nil
+}
+
+func generateReadmeData(license string) string {
+ return `
+---
+license: ` + license + `
+---
+ `
+}
+
+func (c *datasetComponentImpl) Index(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Dataset, int, error) {
+ return c.commonIndex(ctx, filter, per, page)
+}
+
+func (c *datasetComponentImpl) commonIndex(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Dataset, int, error) {
+ var (
+ err error
+ resDatasets []types.Dataset
+ )
+ repos, total, err := c.repoComponent.PublicToUser(ctx, types.DatasetRepo, filter.Username, filter, per, page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public dataset repos,error:%w", err)
+ return nil, 0, newError
+ }
+ var repoIDs []int64
+ for _, repo := range repos {
+ repoIDs = append(repoIDs, repo.ID)
+ }
+ datasets, err := c.datasetStore.ByRepoIDs(ctx, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get datasets by repo ids,error:%w", err)
+ return nil, 0, newError
+ }
+
+ // loop through repos to keep the repos in sort order
+ for _, repo := range repos {
+ var dataset *database.Dataset
+ for _, d := range datasets {
+ if repo.ID == d.RepositoryID {
+ dataset = &d
+ break
+ }
+ }
+ if dataset == nil {
+ continue
+ }
+ var tags []types.RepoTag
+ for _, tag := range repo.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ resDatasets = append(resDatasets, types.Dataset{
+ ID: dataset.ID,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ Likes: repo.Likes,
+ Downloads: repo.DownloadCount,
+ Path: repo.Path,
+ RepositoryID: repo.ID,
+ Private: repo.Private,
+ Tags: tags,
+ CreatedAt: dataset.CreatedAt,
+ UpdatedAt: repo.UpdatedAt,
+ Source: repo.Source,
+ SyncStatus: repo.SyncStatus,
+ License: repo.License,
+ Repository: common.BuildCloneInfo(c.config, dataset.Repository),
+ User: types.User{
+ Username: dataset.Repository.User.Username,
+ Nickname: dataset.Repository.User.NickName,
+ Email: dataset.Repository.User.Email,
+ Avatar: dataset.Repository.User.Avatar,
+ },
+ })
+ }
+
+ return resDatasets, total, nil
+}
+
+func (c *datasetComponentImpl) Update(ctx context.Context, req *types.UpdateDatasetReq) (*types.Dataset, error) {
+ req.RepoType = types.DatasetRepo
+ dbRepo, err := c.repoComponent.UpdateRepo(ctx, req.UpdateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ dataset, err := c.datasetStore.ByRepoID(ctx, dbRepo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ // update times of dateset
+ err = c.datasetStore.Update(ctx, *dataset)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update database dataset, error: %w", err)
+ }
+
+ resDataset := &types.Dataset{
+ ID: dataset.ID,
+ Name: dbRepo.Name,
+ Nickname: dbRepo.Nickname,
+ Description: dbRepo.Description,
+ Likes: dbRepo.Likes,
+ Downloads: dbRepo.DownloadCount,
+ Path: dbRepo.Path,
+ RepositoryID: dbRepo.ID,
+ Private: dbRepo.Private,
+ CreatedAt: dataset.CreatedAt,
+ UpdatedAt: dataset.UpdatedAt,
+ }
+
+ return resDataset, nil
+}
+
+func (c *datasetComponentImpl) Delete(ctx context.Context, namespace, name, currentUser string) error {
+ dataset, err := c.datasetStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ deleteDatabaseRepoReq := types.DeleteRepoReq{
+ Username: currentUser,
+ Namespace: namespace,
+ Name: name,
+ RepoType: types.DatasetRepo,
+ }
+ _, err = c.repoComponent.DeleteRepo(ctx, deleteDatabaseRepoReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete repo of dataset, error: %w", err)
+ }
+
+ err = c.datasetStore.Delete(ctx, *dataset)
+ if err != nil {
+ return fmt.Errorf("failed to delete database dataset, error: %w", err)
+ }
+ return nil
+}
+
+func (c *datasetComponentImpl) Show(ctx context.Context, namespace, name, currentUser string) (*types.Dataset, error) {
+ var tags []types.RepoTag
+ dataset, err := c.datasetStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, currentUser, dataset.Repository)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ ns, err := c.repoComponent.GetNameSpaceInfo(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace info for dataset, error: %w", err)
+ }
+
+ for _, tag := range dataset.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ likeExists, err := c.userLikesStore.IsExist(ctx, currentUser, dataset.Repository.ID)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user likes,error:%w", err)
+ return nil, newError
+ }
+
+ resDataset := &types.Dataset{
+ ID: dataset.ID,
+ Name: dataset.Repository.Name,
+ Nickname: dataset.Repository.Nickname,
+ Description: dataset.Repository.Description,
+ Likes: dataset.Repository.Likes,
+ Downloads: dataset.Repository.DownloadCount,
+ Path: dataset.Repository.Path,
+ RepositoryID: dataset.Repository.ID,
+ DefaultBranch: dataset.Repository.DefaultBranch,
+ Repository: common.BuildCloneInfo(c.config, dataset.Repository),
+ Tags: tags,
+ User: types.User{
+ Username: dataset.Repository.User.Username,
+ Nickname: dataset.Repository.User.NickName,
+ Email: dataset.Repository.User.Email,
+ Avatar: dataset.Repository.User.Avatar,
+ },
+ Private: dataset.Repository.Private,
+ CreatedAt: dataset.CreatedAt,
+ UpdatedAt: dataset.Repository.UpdatedAt,
+ UserLikes: likeExists,
+ Source: dataset.Repository.Source,
+ SyncStatus: dataset.Repository.SyncStatus,
+ License: dataset.Repository.License,
+ MirrorLastUpdatedAt: dataset.Repository.Mirror.LastUpdatedAt,
+ CanWrite: permission.CanWrite,
+ CanManage: permission.CanAdmin,
+ Namespace: ns,
+ }
+ if permission.CanAdmin {
+ resDataset.SensitiveCheckStatus = dataset.Repository.SensitiveCheckStatus.String()
+ }
+
+ return resDataset, nil
+}
+
+func (c *datasetComponentImpl) Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error) {
+ dataset, err := c.datasetStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset repo, error: %w", err)
+ }
+
+ allow, _ := c.repoComponent.AllowReadAccessRepo(ctx, dataset.Repository, currentUser)
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ return c.getRelations(ctx, dataset.RepositoryID, currentUser)
+}
+
+func (c *datasetComponentImpl) getRelations(ctx context.Context, repoID int64, currentUser string) (*types.Relations, error) {
+ res, err := c.repoComponent.RelatedRepos(ctx, repoID, currentUser)
+ if err != nil {
+ return nil, err
+ }
+ rels := new(types.Relations)
+ modelRepos := res[types.ModelRepo]
+ for _, repo := range modelRepos {
+ rels.Models = append(rels.Models, &types.Model{
+ Path: repo.Path,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ UpdatedAt: repo.UpdatedAt,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ })
+ }
+
+ return rels, nil
+}
+
+func (c *datasetComponentImpl) OrgDatasets(ctx context.Context, req *types.OrgDatasetsReq) ([]types.Dataset, int, error) {
+ var resDatasets []types.Dataset
+ var err error
+ r := membership.RoleUnknown
+ if req.CurrentUser != "" {
+ r, err = c.userSvcClient.GetMemberRole(ctx, req.Namespace, req.CurrentUser)
+ // log error, and treat user as unknown role in org
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Namespace), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ }
+ onlyPublic := !r.CanRead()
+ datasets, total, err := c.datasetStore.ByOrgPath(ctx, req.Namespace, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user datasets,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range datasets {
+ resDatasets = append(resDatasets, types.Dataset{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resDatasets, total, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/base64"
+ "fmt"
+ "log/slog"
+ "regexp"
+ "strings"
+
+ "gopkg.in/yaml.v3"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/parquet"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var (
+ REPOCARD_FILENAME = "README.md"
+ WILDCARD = "*"
+ REG = regexp.MustCompile(`(?s)---\n(.*?)\n---`)
+)
+
+type ViewParquetFileReq struct {
+ Namespace string `json:"namespace"`
+ RepoName string `json:"name"`
+ Branch string `json:"branch"`
+ Path string `json:"path"`
+ Per int `json:"per"`
+ Page int `json:"page"`
+ CurrentUser string `json:"current_user"`
+}
+
+type ViewParquetFileResp struct {
+ Columns []string `json:"columns"`
+ ColumnsType []string `json:"columns_type"`
+ Rows [][]interface{} `json:"rows"`
+ Total int `json:"total"`
+ Orderby string `json:"orderby"`
+ Where string `json:"where"`
+ Search string `json:"search"`
+}
+
+type datasetViewerComponentImpl struct {
+ repoStore database.RepoStore
+ repoComponent RepoComponent
+ gitServer gitserver.GitServer
+ preader parquet.Reader
+ cfg *config.Config
+}
+
+type CardData struct {
+ Configs []ConfigData `yaml:"configs" json:"configs"`
+ DatasetInfos []DatasetInfo `yaml:"dataset_info" json:"dataset_info"`
+}
+
+type ConfigData struct {
+ ConfigName string `yaml:"config_name" json:"config_name"`
+ DataFiles []DataFiles `yaml:"data_files" json:"data_files"`
+}
+
+type DataFiles struct {
+ Split string `yaml:"split" json:"split"`
+ Path interface{} `yaml:"path" json:"path"`
+}
+
+type DatasetInfo struct {
+ ConfigName string `yaml:"config_name" json:"config_name"`
+ Splits []Split `yaml:"splits" json:"splits"`
+}
+
+type Split struct {
+ Name string `yaml:"name" json:"name"`
+ NumExamples int `yaml:"num_examples" json:"num_examples"`
+}
+
+type DatasetViewerComponent interface {
+ ViewParquetFile(ctx context.Context, req *ViewParquetFileReq) (*ViewParquetFileResp, error)
+ Rows(ctx context.Context, req *ViewParquetFileReq, viewerReq types.DataViewerReq) (*ViewParquetFileResp, error)
+ GetCatalog(ctx context.Context, req *ViewParquetFileReq) (*CardData, error)
+}
+
+func NewDatasetViewerComponent(cfg *config.Config) (DatasetViewerComponent, error) {
+ r, err := NewRepoComponentImpl(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create repo component,cause:%w", err)
+ }
+ gs, err := git.NewGitServer(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server,cause:%w", err)
+ }
+
+ return &datasetViewerComponentImpl{
+ gitServer: gs,
+ cfg: cfg,
+ repoComponent: r,
+ repoStore: database.NewRepoStore(),
+ }, nil
+}
+
+func (c *datasetViewerComponentImpl) lazyInit() error {
+ if c.preader != nil {
+ return nil
+ }
+ r, err := parquet.NewS3Reader(c.cfg)
+ if err != nil {
+ c.preader = nil
+ return fmt.Errorf("failed to create parquet reader,cause: %w", err)
+ }
+ c.preader = r
+ return nil
+}
+
+func (c *datasetViewerComponentImpl) ViewParquetFile(ctx context.Context, req *ViewParquetFileReq) (*ViewParquetFileResp, error) {
+ err := c.lazyInit()
+ if err != nil {
+ return nil, fmt.Errorf("failed to init parquet reader, %w", err)
+ }
+
+ r, err := c.repoStore.FindByPath(ctx, types.DatasetRepo, req.Namespace, req.RepoName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ allow, err := c.repoComponent.AllowReadAccessRepo(ctx, r, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check dataset permission, error: %w", err)
+ }
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ objName, err := c.getParquetObject(req)
+ if err != nil {
+ return nil, fmt.Errorf("no valid parquet file in request, %w", err)
+ }
+
+ sqlReq := types.QueryReq{
+ PageSize: req.Per,
+ PageIndex: req.Page,
+ Search: "",
+ Where: "",
+ Orderby: "",
+ }
+
+ total, err := c.preader.RowCount([]string{objName}, sqlReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get total count of parquet file, %w", err)
+ }
+
+ columns, columnsType, rows, err := c.preader.FetchRows([]string{objName}, sqlReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to view parquet rows, %w", err)
+ }
+
+ resp := &ViewParquetFileResp{
+ Columns: columns,
+ ColumnsType: columnsType,
+ Rows: rows,
+ Total: total,
+ }
+ return resp, nil
+}
+
+func (c *datasetViewerComponentImpl) Rows(ctx context.Context, req *ViewParquetFileReq, viewerReq types.DataViewerReq) (*ViewParquetFileResp, error) {
+ err := c.lazyInit()
+ if err != nil {
+ return nil, fmt.Errorf("failed to init parquet reader, %w", err)
+ }
+
+ r, err := c.repoStore.FindByPath(ctx, types.DatasetRepo, req.Namespace, req.RepoName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ allow, err := c.repoComponent.AllowReadAccessRepo(ctx, r, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check dataset permission, error: %w", err)
+ }
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ cardData, err := c.getDatasetCatalog(req, false)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get dataset catalog, error: %w", err)
+ }
+ slog.Debug("Rows cardData", slog.Any("cardData", cardData))
+ var reqFiles []string
+ var hasWildCard bool
+ var tree []string
+
+ for _, config := range cardData.Configs {
+ if config.ConfigName == viewerReq.Config {
+ for _, datafile := range config.DataFiles {
+ if datafile.Split == viewerReq.Split {
+ reqFiles, hasWildCard = getViewerFileList(datafile.Path)
+ break
+ }
+ }
+ break
+ }
+ }
+
+ realReqFiles := reqFiles
+ if hasWildCard {
+ // need get real files match test/test-* in repo
+ realReqFiles, _ = c.convertRealFiles(req, reqFiles, tree)
+ }
+ parquetObjs := c.getFilesOBJs(req, realReqFiles)
+ if len(parquetObjs) < 1 {
+ return nil, fmt.Errorf("no valid files in request")
+ }
+ sqlReq := types.QueryReq{
+ PageSize: req.Per,
+ PageIndex: req.Page,
+ Search: viewerReq.Search,
+ Where: viewerReq.Where,
+ Orderby: viewerReq.Orderby,
+ }
+
+ total, err := c.preader.RowCount(parquetObjs, sqlReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get total count of parquet file, %w", err)
+ }
+ columns, columnsType, rows, err := c.preader.FetchRows(parquetObjs, sqlReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to view parquet rows, %w", err)
+ }
+
+ resp := &ViewParquetFileResp{
+ Columns: columns,
+ ColumnsType: columnsType,
+ Rows: rows,
+ Total: total,
+ Orderby: viewerReq.Orderby,
+ Where: viewerReq.Where,
+ Search: viewerReq.Search,
+ }
+ return resp, nil
+}
+
+func getViewerFileList(path interface{}) ([]string, bool) {
+ if path == nil {
+ return []string{}, false
+ }
+ var (
+ files []string
+ hasWildCard bool
+ )
+ hasWildCard = false
+ if slice, ok := path.([]interface{}); ok {
+ for _, v := range slice {
+ files = append(files, v.(string))
+ if strings.HasSuffix(v.(string), WILDCARD) {
+ hasWildCard = true
+ }
+ }
+ } else if slice, ok := path.([]string); ok {
+ files = slice
+ for _, v := range slice {
+ if strings.HasSuffix(v, WILDCARD) {
+ hasWildCard = true
+ break
+ }
+ }
+ } else {
+ files = []string{path.(string)}
+ if strings.HasSuffix(path.(string), WILDCARD) {
+ hasWildCard = true
+ }
+ }
+ return files, hasWildCard
+}
+
+func (c *datasetViewerComponentImpl) GetCatalog(ctx context.Context, req *ViewParquetFileReq) (*CardData, error) {
+ err := c.lazyInit()
+ if err != nil {
+ return nil, fmt.Errorf("failed to init parquet reader, %w", err)
+ }
+
+ r, err := c.repoStore.FindByPath(ctx, types.DatasetRepo, req.Namespace, req.RepoName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ allow, err := c.repoComponent.AllowReadAccessRepo(ctx, r, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check dataset permission, error: %w", err)
+ }
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+ cardData, err := c.getDatasetCatalog(req, true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get dataset catalog, error: %w", err)
+ }
+ return cardData, nil
+}
+
+func (c *datasetViewerComponentImpl) getDatasetCatalog(req *ViewParquetFileReq, calcTotal bool) (*CardData, error) {
+ cardData, err := c.getRepoCardData(req, calcTotal)
+ if err == nil {
+ return cardData, nil
+ }
+ slog.Warn("cannot get card data for repo", slog.Any("req", req), slog.Any("error", err))
+ cardData, err = c.autoGenerateCatalog(req, calcTotal)
+ if err != nil {
+ return nil, fmt.Errorf("failed to auto gen catalog, error: %w", err)
+ }
+ return cardData, nil
+}
+
+func (c *datasetViewerComponentImpl) getRepoCardData(req *ViewParquetFileReq, calcTotal bool) (*CardData, error) {
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.RepoName,
+ Ref: req.Branch,
+ Path: REPOCARD_FILENAME,
+ RepoType: types.DatasetRepo,
+ }
+ f, err := c.gitServer.GetRepoFileContents(context.Background(), getFileContentReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get readme.md contents, cause:%w", err)
+ }
+ slog.Debug("getRepoCardData", slog.Any("f.Content", f.Content))
+ decodedContent, err := base64.StdEncoding.DecodeString(f.Content)
+ if err != nil {
+ return nil, fmt.Errorf("failed to base64 decode readme.md contents, cause:%w", err)
+ }
+ decodedContentStr := string(decodedContent)
+ matches := REG.FindStringSubmatch(decodedContentStr)
+ yamlString := ""
+ if len(matches) > 1 {
+ yamlString = matches[1]
+ } else {
+ return nil, fmt.Errorf("repo card yaml configs is empty")
+ }
+
+ var card CardData
+ err = yaml.Unmarshal([]byte(yamlString), &card)
+ if err != nil {
+ return nil, fmt.Errorf("failed to Unmarshal readme.md yaml contents, cause: %w, decodedContent: %v", err, yamlString)
+ }
+ slog.Debug("Unmarshal", slog.Any("card", card))
+ if card.Configs == nil {
+ return nil, fmt.Errorf("repo card data configs is empty")
+ }
+ if calcTotal && card.DatasetInfos == nil {
+ card = c.generateCardDatasetInfo(req, card)
+ }
+ return &card, nil
+}
+
+func (c *datasetViewerComponentImpl) generateCardDatasetInfo(req *ViewParquetFileReq, card CardData) CardData {
+ var configs []ConfigData
+ var infos []DatasetInfo
+ var tree []string
+ for _, conf := range card.Configs {
+ var datafiles []DataFiles
+ var splits []Split
+ for _, datafile := range conf.DataFiles {
+ var newPath interface{}
+ reqFiles, hasWildCard := getViewerFileList(datafile.Path)
+ if len(reqFiles) > 0 {
+ newPath = reqFiles
+ } else {
+ slog.Warn("datafile.Path is not either string or []interface{})", slog.Any("datafile.Path", datafile.Path))
+ newPath = datafile.Path
+ }
+ datafiles = append(datafiles, DataFiles{Split: datafile.Split, Path: newPath})
+ realReqFiles := reqFiles
+ if hasWildCard {
+ realReqFiles, tree = c.convertRealFiles(req, reqFiles, tree)
+ }
+ total := c.getFilesRowCount(req, realReqFiles)
+ splits = append(splits, Split{Name: datafile.Split, NumExamples: total})
+ }
+ configs = append(configs, ConfigData{ConfigName: conf.ConfigName, DataFiles: datafiles})
+ infos = append(infos, DatasetInfo{ConfigName: conf.ConfigName, Splits: splits})
+ }
+ return CardData{Configs: configs, DatasetInfos: infos}
+}
+
+func (c *datasetViewerComponentImpl) convertRealFiles(req *ViewParquetFileReq, splitFiles []string, tree []string) ([]string, []string) {
+ var err error
+ if len(tree) < 1 {
+ // skip get all tree
+ tree, err = getFilePaths(req.Namespace, req.RepoName, "", types.DatasetRepo, req.Branch, c.gitServer.GetRepoFileTree)
+ if err != nil {
+ slog.Error("Failed to get repo file paths", slog.Any("req", req), slog.Any("error", err))
+ return splitFiles, tree
+ }
+ if tree == nil {
+ return splitFiles, tree
+ }
+ }
+
+ var phyFiles []string
+
+ for _, filePattern := range splitFiles {
+ if !strings.Contains(filePattern, WILDCARD) {
+ phyFiles = append(phyFiles, filePattern)
+ } else {
+ fileReg, err := regexp.Compile(filePattern)
+ if err != nil {
+ slog.Warn("invalid regexp format", slog.Any("filePattern", filePattern), slog.Any("err", err))
+ phyFiles = append(phyFiles, filePattern)
+ continue
+ }
+ for _, repoFile := range tree {
+ // repo file match like: test/test-* and end with .parquet
+ if fileReg.MatchString(repoFile) && isValidFile(repoFile) {
+ phyFiles = append(phyFiles, repoFile)
+ }
+ }
+ }
+ }
+ slog.Debug("convertRealFiles", slog.Any("splitFiles", splitFiles), slog.Any("phyFiles", phyFiles))
+ return phyFiles, tree
+}
+
+func (c *datasetViewerComponentImpl) autoGenerateCatalog(req *ViewParquetFileReq, calcTotal bool) (*CardData, error) {
+ tree, err := getFilePaths(req.Namespace, req.RepoName, "", types.DatasetRepo, req.Branch, c.gitServer.GetRepoFileTree)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get repo tree, error: %w", err)
+ }
+ if tree == nil {
+ return nil, fmt.Errorf("failed to find any files")
+ }
+ slog.Debug("get tree", slog.Any("tree", tree))
+ cardData := c.genDefaultCatalog(req, tree, calcTotal)
+ return &cardData, nil
+}
+
+func (c *datasetViewerComponentImpl) genDefaultCatalog(req *ViewParquetFileReq, tree []string, calcTotal bool) CardData {
+ var (
+ trainFiles []string
+ testFiles []string
+ valFiles []string
+ defaultName = "default"
+ trainSplit = "train"
+ testSplit = "test"
+ valSplit = "validation"
+ )
+ var configData ConfigData
+ var datasetInfo DatasetInfo
+ for _, item := range tree {
+ if !isValidFile(item) {
+ continue
+ }
+ if isTrainFile(strings.ToLower(item)) {
+ trainFiles = append(trainFiles, item)
+ } else if isTestFile(strings.ToLower(item)) {
+ testFiles = append(testFiles, item)
+ } else if isValidationFile(strings.ToLower(item)) {
+ valFiles = append(valFiles, item)
+ }
+ }
+ if len(trainFiles) > 0 {
+ total := 0
+ if calcTotal {
+ total = c.getFilesRowCount(req, trainFiles)
+ }
+ configData.DataFiles = append(configData.DataFiles, DataFiles{Split: trainSplit, Path: trainFiles})
+ datasetInfo.Splits = append(datasetInfo.Splits, Split{Name: trainSplit, NumExamples: total})
+ }
+ if len(testFiles) > 0 {
+ total := 0
+ if calcTotal {
+ total = c.getFilesRowCount(req, testFiles)
+ }
+ configData.DataFiles = append(configData.DataFiles, DataFiles{Split: testSplit, Path: testFiles})
+ datasetInfo.Splits = append(datasetInfo.Splits, Split{Name: testSplit, NumExamples: total})
+ }
+ if len(valFiles) > 0 {
+ total := 0
+ if calcTotal {
+ total = c.getFilesRowCount(req, valFiles)
+ }
+ configData.DataFiles = append(configData.DataFiles, DataFiles{Split: valSplit, Path: valFiles})
+ datasetInfo.Splits = append(datasetInfo.Splits, Split{Name: valSplit, NumExamples: total})
+ }
+ configData.ConfigName = defaultName
+ datasetInfo.ConfigName = defaultName
+ var configList []ConfigData
+ var dsInfoList []DatasetInfo
+ if len(configData.DataFiles) > 0 {
+ configList = append(configList, configData)
+ dsInfoList = append(dsInfoList, datasetInfo)
+ }
+ return CardData{Configs: configList, DatasetInfos: dsInfoList}
+}
+
+func (c *datasetViewerComponentImpl) getFilesRowCount(req *ViewParquetFileReq, files []string) int {
+ slog.Debug("getFilesRowCount", slog.Any("files", files))
+ parquetObjs := c.getFilesOBJs(req, files)
+ if len(parquetObjs) < 1 {
+ return 0
+ }
+ sqlReq := types.QueryReq{
+ PageSize: 10,
+ PageIndex: 1,
+ Search: "",
+ Orderby: "",
+ }
+
+ total, err := c.preader.RowCount(parquetObjs, sqlReq)
+ if err != nil {
+ slog.Warn("failed to get parquet row counts", slog.Any("parquetObjs", parquetObjs), slog.Any("error", err))
+ }
+ return total
+}
+
+func (c *datasetViewerComponentImpl) getFilesOBJs(req *ViewParquetFileReq, files []string) []string {
+ var parquetObjs []string
+ for _, file := range files {
+ objName, err := c.getParquetObject(&ViewParquetFileReq{
+ Namespace: req.Namespace,
+ RepoName: req.RepoName,
+ Branch: req.Branch,
+ Path: file,
+ })
+ if err != nil {
+ slog.Warn("failed to get parquet object name", slog.Any("req", req), slog.Any("error", err))
+ continue
+ }
+ parquetObjs = append(parquetObjs, objName)
+ }
+ return parquetObjs
+}
+
+func (c *datasetViewerComponentImpl) getParquetObject(req *ViewParquetFileReq) (string, error) {
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.RepoName,
+ Ref: req.Branch,
+ Path: req.Path,
+ RepoType: types.DatasetRepo,
+ }
+ f, err := c.gitServer.GetRepoFileContents(context.Background(), getFileContentReq)
+ if err != nil {
+ return "", fmt.Errorf("failed to get file contents,cause:%v", err)
+ }
+ if f.LfsRelativePath == "" {
+ return "", fmt.Errorf("file LfsRelativePath is empty for %s", getFileContentReq.Path)
+ }
+ return "lfs/" + f.LfsRelativePath, nil
+}
+
+func isValidFile(entry string) bool {
+ return strings.HasSuffix(strings.ToLower(entry), ".parquet")
+}
+
+func isTrainFile(fileName string) bool {
+ if strings.Contains(fileName, "train") || strings.Contains(fileName, "training") {
+ return true
+ }
+ return false
+}
+
+func isTestFile(fileName string) bool {
+ if strings.Contains(fileName, "test") || strings.Contains(fileName, "testing") {
+ return true
+ }
+ if strings.Contains(fileName, "eval") || strings.Contains(fileName, "evaluation") {
+ return true
+ }
+ return false
+}
+
+func isValidationFile(fileName string) bool {
+ if strings.Contains(fileName, "val") || strings.Contains(fileName, "valid") || strings.Contains(fileName, "validation") {
+ return true
+ }
+ if strings.Contains(fileName, "dev") {
+ return true
+ }
+ return false
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "time"
+
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type discussionComponentImpl struct {
+ discussionStore database.DiscussionStore
+ repoStore database.RepoStore
+ userStore database.UserStore
+}
+
+type DiscussionComponent interface {
+ CreateRepoDiscussion(ctx context.Context, req CreateRepoDiscussionRequest) (*CreateDiscussionResponse, error)
+ GetDiscussion(ctx context.Context, id int64) (*ShowDiscussionResponse, error)
+ UpdateDiscussion(ctx context.Context, req UpdateDiscussionRequest) error
+ DeleteDiscussion(ctx context.Context, currentUser string, id int64) error
+ ListRepoDiscussions(ctx context.Context, req ListRepoDiscussionRequest) (*ListRepoDiscussionResponse, error)
+ CreateDiscussionComment(ctx context.Context, req CreateCommentRequest) (*CreateCommentResponse, error)
+ UpdateComment(ctx context.Context, currentUser string, id int64, content string) error
+ DeleteComment(ctx context.Context, currentUser string, id int64) error
+ ListDiscussionComments(ctx context.Context, discussionID int64) ([]*DiscussionResponse_Comment, error)
+}
+
+func NewDiscussionComponent() DiscussionComponent {
+ ds := database.NewDiscussionStore()
+ rs := database.NewRepoStore()
+ us := database.NewUserStore()
+ return &discussionComponentImpl{discussionStore: ds, repoStore: rs, userStore: us}
+}
+
+func (c *discussionComponentImpl) CreateRepoDiscussion(ctx context.Context, req CreateRepoDiscussionRequest) (*CreateDiscussionResponse, error) {
+ //TODO:check if the user can access the repo
+
+ //get repo by namespace and name
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo by path '%s/%s/%s': %w", req.RepoType, req.Namespace, req.Name, err)
+ }
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user by username '%s': %w", req.CurrentUser, err)
+ }
+ discussion, err := c.discussionStore.Create(ctx, database.Discussion{
+ Title: req.Title,
+ DiscussionableID: repo.ID,
+ DiscussionableType: database.DiscussionableTypeRepo,
+ UserID: user.ID,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create discussion: %w", err)
+ }
+ resp := &CreateDiscussionResponse{
+ ID: discussion.ID,
+ User: &DiscussionResponse_User{
+ ID: user.ID,
+ Username: user.Username,
+ Avatar: user.Avatar,
+ },
+ Title: discussion.Title,
+ CommentCount: discussion.CommentCount,
+ CreatedAt: discussion.CreatedAt,
+ }
+ return resp, nil
+}
+
+func (c *discussionComponentImpl) GetDiscussion(ctx context.Context, id int64) (*ShowDiscussionResponse, error) {
+ discussion, err := c.discussionStore.FindByID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find discussion by id '%d': %w", id, err)
+ }
+ comments, err := c.discussionStore.FindDiscussionComments(ctx, discussion.ID)
+ if err != nil && err != sql.ErrNoRows {
+ return nil, fmt.Errorf("failed to find discussion comments by discussion id '%d': %w", discussion.ID, err)
+ }
+ resp := &ShowDiscussionResponse{
+ ID: discussion.ID,
+ Title: discussion.Title,
+ User: &DiscussionResponse_User{
+ ID: discussion.User.ID,
+ Username: discussion.User.Username,
+ Avatar: discussion.User.Avatar,
+ },
+ }
+ for _, comment := range comments {
+ resp.Comments = append(resp.Comments, &DiscussionResponse_Comment{
+ ID: comment.ID,
+ Content: comment.Content,
+ User: &DiscussionResponse_User{
+ ID: comment.User.ID,
+ Username: comment.User.Username,
+ Avatar: comment.User.Avatar,
+ },
+ })
+ }
+ return resp, nil
+}
+
+func (c *discussionComponentImpl) UpdateDiscussion(ctx context.Context, req UpdateDiscussionRequest) error {
+ //check if the user is the owner of the discussion
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("failed to find user by username '%s': %w", req.CurrentUser, err)
+ }
+ discussion, err := c.discussionStore.FindByID(ctx, req.ID)
+ if err != nil {
+ return fmt.Errorf("failed to find discussion by id '%d': %w", req.ID, err)
+ }
+ if discussion.UserID != user.ID {
+ return fmt.Errorf("user '%s' is not the owner of the discussion '%d'", req.CurrentUser, req.ID)
+ }
+ err = c.discussionStore.UpdateByID(ctx, req.ID, req.Title)
+ if err != nil {
+ return fmt.Errorf("failed to update discussion by id '%d': %w", req.ID, err)
+ }
+ return nil
+}
+
+func (c *discussionComponentImpl) DeleteDiscussion(ctx context.Context, currentUser string, id int64) error {
+ discussion, err := c.discussionStore.FindByID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to find discussion by id '%d': %w", id, err)
+ }
+ if discussion.User.Username != currentUser {
+ return fmt.Errorf("user '%s' is not the owner of the discussion '%d'", currentUser, id)
+ }
+ err = c.discussionStore.DeleteByID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to delete discussion by id '%d': %w", id, err)
+ }
+ return nil
+}
+
+func (c *discussionComponentImpl) ListRepoDiscussions(ctx context.Context, req ListRepoDiscussionRequest) (*ListRepoDiscussionResponse, error) {
+ //TODO:check if the user can access the repo
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo by path '%s/%s/%s': %w", req.RepoType, req.Namespace, req.Name, err)
+ }
+ discussions, err := c.discussionStore.FindByDiscussionableID(ctx, database.DiscussionableTypeRepo, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list repo discussions by repo type '%s', namespace '%s', name '%s': %w", req.RepoType, req.Namespace, req.Name, err)
+ }
+ resp := &ListRepoDiscussionResponse{}
+ for _, discussion := range discussions {
+ resp.Discussions = append(resp.Discussions, &CreateDiscussionResponse{
+ ID: discussion.ID,
+ Title: discussion.Title,
+ CommentCount: discussion.CommentCount,
+ CreatedAt: discussion.CreatedAt,
+ User: &DiscussionResponse_User{
+ ID: discussion.User.ID,
+ Username: discussion.User.Username,
+ Avatar: discussion.User.Avatar,
+ },
+ })
+ }
+ return resp, nil
+}
+
+func (c *discussionComponentImpl) CreateDiscussionComment(ctx context.Context, req CreateCommentRequest) (*CreateCommentResponse, error) {
+ req.CommentableType = database.CommentableTypeDiscussion
+ // get discussion by id
+ _, err := c.discussionStore.FindByID(ctx, req.CommentableID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find discussion by id '%d': %w", req.CommentableID, err)
+ }
+
+ //get user by username
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user by username '%s': %w", req.CurrentUser, err)
+ }
+ // create comment
+ comment, err := c.discussionStore.CreateComment(ctx, database.Comment{
+ Content: req.Content,
+ CommentableID: req.CommentableID,
+ CommentableType: req.CommentableType,
+ UserID: user.ID,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create discussion comment: %w", err)
+ }
+ return &CreateCommentResponse{
+ ID: comment.ID,
+ CommentableID: comment.CommentableID,
+ CommentableType: comment.CommentableType,
+ CreatedAt: comment.CreatedAt,
+ User: &DiscussionResponse_User{
+ ID: user.ID,
+ Username: user.Username,
+ Avatar: user.Avatar,
+ },
+ }, nil
+}
+
+func (c *discussionComponentImpl) UpdateComment(ctx context.Context, currentUser string, id int64, content string) error {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return fmt.Errorf("failed to find user by username '%s': %w", currentUser, err)
+ }
+ //get comment by id
+ comment, err := c.discussionStore.FindCommentByID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to find comment by id '%d': %w", id, err)
+ }
+ //check if the user is the owner of the comment
+ if comment.UserID != user.ID {
+ return fmt.Errorf("user '%s' is not the owner of the comment '%d'", currentUser, id)
+ }
+ err = c.discussionStore.UpdateComment(ctx, id, content)
+ if err != nil {
+ return fmt.Errorf("failed to update comment by id '%d': %w", id, err)
+ }
+ return nil
+}
+
+func (c *discussionComponentImpl) DeleteComment(ctx context.Context, currentUser string, id int64) error {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return fmt.Errorf("failed to find user by username '%s': %w", currentUser, err)
+ }
+ //get comment by id
+ comment, err := c.discussionStore.FindCommentByID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to find comment by id '%d': %w", id, err)
+ }
+ //check if the user is the owner of the comment
+ if comment.UserID != user.ID {
+ return fmt.Errorf("user '%s' is not the owner of the comment '%d'", currentUser, id)
+ }
+ err = c.discussionStore.DeleteComment(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to delete comment by id '%d': %w", id, err)
+ }
+ return nil
+}
+
+func (c *discussionComponentImpl) ListDiscussionComments(ctx context.Context, discussionID int64) ([]*DiscussionResponse_Comment, error) {
+ comments, err := c.discussionStore.FindDiscussionComments(ctx, discussionID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find discussion comments by discussion id '%d': %w", discussionID, err)
+ }
+ resp := make([]*DiscussionResponse_Comment, 0, len(comments))
+ for _, comment := range comments {
+ resp = append(resp, &DiscussionResponse_Comment{
+ ID: comment.ID,
+ Content: comment.Content,
+ User: &DiscussionResponse_User{
+ ID: comment.User.ID,
+ Username: comment.User.Username,
+ Avatar: comment.User.Avatar,
+ },
+ CreatedAt: comment.CreatedAt,
+ })
+ }
+ return resp, nil
+}
+
+//--- request and response ---//
+
+type CreateRepoDiscussionRequest struct {
+ Title string `json:"title" binding:"required"`
+ RepoType types.RepositoryType `json:"-"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+// CreateRepoDiscussionRequest implements types.SensitiveRequestV2
+var _ types.SensitiveRequestV2 = (*CreateRepoDiscussionRequest)(nil)
+
+func (req *CreateRepoDiscussionRequest) GetSensitiveFields() []types.SensitiveField {
+ return []types.SensitiveField{
+ {
+ Name: "title",
+ Value: func() string {
+ return req.Title
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ },
+ }
+}
+
+type CreateDiscussionResponse struct {
+ ID int64 `json:"id"`
+ User *DiscussionResponse_User `json:"user"`
+ Title string `json:"title"`
+ // DiscussionableID int64 `json:"discussionable_id"`
+ // DiscussionableType string `json:"discussionable_type"`
+ CommentCount int64 `json:"comment_count"`
+ CreatedAt time.Time `json:"created_at"`
+ // UpdatedAt time.Time `json:"updated_at"`
+}
+
+type DiscussionResponse_User struct {
+ ID int64 `json:"id"`
+ Username string `json:"name"`
+ Avatar string `json:"avatar"`
+}
+
+type UpdateDiscussionRequest struct {
+ ID int64 `json:"-"`
+ Title string `json:"title" binding:"required"`
+ CurrentUser string `json:"-"`
+}
+
+// UpdateDiscussionRequest implements types.SensitiveRequestV2
+var _ types.SensitiveRequestV2 = (*UpdateDiscussionRequest)(nil)
+
+func (req *UpdateDiscussionRequest) GetSensitiveFields() []types.SensitiveField {
+ return []types.SensitiveField{
+ {
+ Name: "title",
+ Value: func() string {
+ return req.Title
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ },
+ }
+}
+
+type ShowDiscussionResponse struct {
+ ID int64 `json:"id"`
+ Title string `json:"title"`
+ User *DiscussionResponse_User `json:"user"`
+ CommentCount int64 `json:"comment_count"`
+ Comments []*DiscussionResponse_Comment `json:"comments,omitempty"`
+}
+
+type DiscussionResponse_Comment struct {
+ ID int64 `json:"id"`
+ Content string `json:"content"`
+ User *DiscussionResponse_User `json:"user"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+type ListRepoDiscussionRequest struct {
+ RepoType types.RepositoryType `json:"-"`
+ Namespace string `json:"-"`
+ Name string `json:"-"`
+ CurrentUser string `json:"-"`
+}
+
+type ListRepoDiscussionResponse struct {
+ Discussions []*CreateDiscussionResponse `json:"discussions"`
+}
+
+type CreateCommentRequest struct {
+ Content string `json:"content" binding:"required"`
+ CommentableID int64 `json:"commentable_id"`
+ CommentableType string `json:"commentable_type"`
+ CurrentUser string `json:"-"`
+}
+
+// CreateCommentRequest implements types.SensitiveRequestV2
+var _ types.SensitiveRequestV2 = (*CreateCommentRequest)(nil)
+
+func (req *CreateCommentRequest) GetSensitiveFields() []types.SensitiveField {
+ return []types.SensitiveField{
+ {
+ Name: "content",
+ Value: func() string {
+ return req.Content
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ },
+ }
+}
+
+type CreateCommentResponse struct {
+ ID int64 `json:"id"`
+ CommentableID int64 `json:"commentable_id"`
+ CommentableType string `json:"commentable_type"`
+ CreatedAt time.Time `json:"created_at"`
+ User *DiscussionResponse_User `json:"user"`
+}
+
+type UpdateCommentRequest struct {
+ ID int64 `json:"-"`
+ Content string `json:"content" binding:"required"`
+}
+
+// UpdateCommentRequest implements types.SensitiveRequestV2
+var _ types.SensitiveRequestV2 = (*UpdateCommentRequest)(nil)
+
+func (req *UpdateCommentRequest) GetSensitiveFields() []types.SensitiveField {
+ return []types.SensitiveField{
+ {
+ Name: "content",
+ Value: func() string {
+ return req.Content
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ },
+ }
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "strconv"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type evaluationComponentImpl struct {
+ deployer deploy.Deployer
+ userStore database.UserStore
+ modelStore database.ModelStore
+ datasetStore database.DatasetStore
+ mirrorStore database.MirrorStore
+ spaceResourceStore database.SpaceResourceStore
+ tokenStore database.AccessTokenStore
+ rtfm database.RuntimeFrameworksStore
+ config *config.Config
+ ac AccountingComponent
+}
+
+type EvaluationComponent interface {
+ // Create argo workflow
+ CreateEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error)
+ GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error)
+ DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error
+}
+
+func NewEvaluationComponent(config *config.Config) (EvaluationComponent, error) {
+ c := &evaluationComponentImpl{}
+ c.deployer = deploy.NewDeployer()
+ c.userStore = database.NewUserStore()
+ c.modelStore = database.NewModelStore()
+ c.spaceResourceStore = database.NewSpaceResourceStore()
+ c.datasetStore = database.NewDatasetStore()
+ c.mirrorStore = database.NewMirrorStore()
+ c.tokenStore = database.NewAccessTokenStore()
+ c.rtfm = database.NewRuntimeFrameworksStore()
+ c.config = config
+ ac, err := NewAccountingComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create accounting component, %w", err)
+ }
+ c.ac = ac
+ return c, nil
+}
+
+// Create argo workflow
+func (c *evaluationComponentImpl) CreateEvaluation(ctx context.Context, req types.EvaluationReq) (*types.ArgoWorkFlowRes, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get current user %s, error:%w", req.Username, err)
+ }
+ result := strings.Split(req.ModelId, "/")
+ _, err = c.modelStore.FindByPath(ctx, result[0], result[1])
+ if err != nil {
+ return nil, fmt.Errorf("cannot find model, %w", err)
+ }
+
+ token, err := c.tokenStore.FindByUID(ctx, user.ID)
+ if err != nil {
+ return nil, fmt.Errorf("cant get git access token:%w", err)
+ }
+ mirrorRepos, err := c.GenerateMirrorRepoIds(ctx, req.Datasets)
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate mirror repo ids, %w", err)
+ }
+ req.Datasets = mirrorRepos
+ req.Token = token.Token
+ var hardware types.HardWare
+ if req.ResourceId != 0 {
+ // check user balance
+ priceData, err := c.ac.QueryPricesBySKUType("", types.AcctPriceListReq{
+ ResourceID: strconv.FormatInt(int64(req.ResourceId), 10),
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("fail to find price data, %w", err)
+ }
+ if priceData.Total == 0 {
+ return nil, fmt.Errorf("cannot find valid price data")
+ }
+
+ if priceData.Prices[0].SkuPrice > 0 {
+ account, err := c.ac.QueryBalanceByUserIDInternal(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find user balance, %w", err)
+ }
+ if account.Balance <= 0 {
+ return nil, fmt.Errorf("balance is not enough to start resources. current balance: %.2f", account.Balance/100)
+ }
+ }
+ resource, err := c.spaceResourceStore.FindByID(ctx, req.ResourceId)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find resource, %w", err)
+ }
+
+ err = json.Unmarshal([]byte(resource.Resources), &hardware)
+ if err != nil {
+ return nil, fmt.Errorf("invalid hardware setting, %w", err)
+ }
+ if hardware.Npu.Num == "" && hardware.Gpu.Num == "" {
+ return nil, fmt.Errorf("evaluation requires GPU or NPU resources")
+ }
+ req.ClusterID = resource.ClusterID
+ req.ResourceName = resource.Name
+ } else {
+ // for share mode
+ hardware.Gpu.Num = c.config.Argo.QuotaGPUNumber
+ hardware.Gpu.ResourceName = "nvidia.com/gpu"
+ hardware.Cpu.Num = "8"
+ hardware.Memory = "32Gi"
+ }
+ frame, err := c.rtfm.FindEnabledByID(ctx, req.RuntimeFrameworkId)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find available runtime framework, %w", err)
+ }
+ req.Hardware = hardware
+ // choose image
+ containerImg := frame.FrameImage
+ if hardware.Npu.Num != "" {
+ // use npu image
+ containerImg = frame.FrameNpuImage
+ }
+
+ req.UserUUID = user.UUID
+ req.Image = containerImg
+ req.RepoType = string(types.ModelRepo)
+ req.TaskType = types.TaskTypeEvaluation
+ req.DownloadEndpoint = c.config.Model.DownloadEndpoint
+ return c.deployer.SubmitEvaluation(ctx, req)
+}
+
+// generate mirror repo ids
+func (c *evaluationComponentImpl) GenerateMirrorRepoIds(ctx context.Context, datasets []string) ([]string, error) {
+ var mirrorRepos []string
+ for _, ds := range datasets {
+ namespace := strings.Split(ds, "/")[0]
+ name := strings.Split(ds, "/")[1]
+ mirrorRepo, err := c.mirrorStore.FindByRepoPath(ctx, types.DatasetRepo, namespace, name)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ //no mirror, will use csghub repo
+ mirrorRepos = append(mirrorRepos, ds)
+ continue
+ }
+ return nil, fmt.Errorf("fail to get mirror repo, %w", err)
+ }
+ mirrorRepos = append(mirrorRepos, mirrorRepo.SourceRepoPath)
+ }
+ return mirrorRepos, nil
+}
+
+func (c *evaluationComponentImpl) DeleteEvaluation(ctx context.Context, req types.ArgoWorkFlowDeleteReq) error {
+ return c.deployer.DeleteEvaluation(ctx, req)
+}
+
+// get evaluation result
+func (c *evaluationComponentImpl) GetEvaluation(ctx context.Context, req types.EvaluationGetReq) (*types.EvaluationRes, error) {
+ wf, err := c.deployer.GetEvaluation(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get evaluation result, %w", err)
+ }
+ datasets, err := c.datasetStore.ListByPath(ctx, wf.Datasets)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get datasets for evaluation, %w", err)
+ }
+ var repoTags []types.RepoTags
+ for _, ds := range datasets {
+ var tags []types.RepoTag
+ for _, tag := range ds.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ var dsRepoTags = types.RepoTags{
+ RepoId: ds.Repository.Path,
+ Tags: tags,
+ }
+ repoTags = append(repoTags, dsRepoTags)
+ }
+ var res = &types.EvaluationRes{
+ ID: wf.ID,
+ RepoIds: wf.RepoIds,
+ RepoType: wf.RepoType,
+ Username: wf.Username,
+ TaskName: wf.TaskName,
+ TaskId: wf.TaskId,
+ TaskType: wf.TaskType,
+ TaskDesc: wf.TaskDesc,
+ ResourceId: wf.ResourceId,
+ Status: string(wf.Status),
+ Reason: wf.Reason,
+ Datasets: repoTags,
+ Image: wf.Image,
+ SubmitTime: wf.SubmitTime,
+ StartTime: wf.StartTime,
+ EndTime: wf.EndTime,
+ ResultURL: wf.ResultURL,
+ DownloadURL: wf.DownloadURL,
+ FailuresURL: wf.FailuresURL,
+ }
+ return res, nil
+}
+
+
+
package component
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type eventComponentImpl struct {
+ es database.EventStore
+}
+
+// NewEventComponent creates a new EventComponent
+
+type EventComponent interface {
+ NewEvents(ctx context.Context, events []types.Event) error
+}
+
+func NewEventComponent() EventComponent {
+ return &eventComponentImpl{
+ es: database.NewEventStore(),
+ }
+}
+
+func (ec *eventComponentImpl) NewEvents(ctx context.Context, events []types.Event) error {
+ var dbevents []database.Event
+ for _, e := range events {
+ dbevents = append(dbevents, database.Event{
+ Module: e.Module,
+ EventID: e.ID,
+ Value: e.Value,
+ Extension: e.Extension,
+ })
+ }
+
+ return ec.es.BatchSave(ctx, dbevents)
+}
+
+
+
package component
+
+import (
+ "context"
+ "crypto/sha256"
+ "database/sql"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "os"
+ "path"
+ "strconv"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/s3"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type gitHTTPComponentImpl struct {
+ gitServer gitserver.GitServer
+ config *config.Config
+ s3Client s3.Client
+ lfsMetaObjectStore database.LfsMetaObjectStore
+ lfsLockStore database.LfsLockStore
+ repoStore database.RepoStore
+ userStore database.UserStore
+ repoComponent RepoComponent
+}
+
+type GitHTTPComponent interface {
+ InfoRefs(ctx context.Context, req types.InfoRefsReq) (io.Reader, error)
+ GitUploadPack(ctx context.Context, req types.GitUploadPackReq) error
+ GitReceivePack(ctx context.Context, req types.GitReceivePackReq) error
+ BuildObjectResponse(ctx context.Context, req types.BatchRequest, isUpload bool) (*types.BatchResponse, error)
+ LfsUpload(ctx context.Context, body io.ReadCloser, req types.UploadRequest) error
+ LfsVerify(ctx context.Context, req types.VerifyRequest, p types.Pointer) error
+ CreateLock(ctx context.Context, req types.LfsLockReq) (*database.LfsLock, error)
+ ListLocks(ctx context.Context, req types.ListLFSLockReq) (*types.LFSLockList, error)
+ UnLock(ctx context.Context, req types.UnlockLFSReq) (*database.LfsLock, error)
+ VerifyLock(ctx context.Context, req types.VerifyLFSLockReq) (*types.LFSLockListVerify, error)
+ LfsDownload(ctx context.Context, req types.DownloadRequest) (*url.URL, error)
+}
+
+func NewGitHTTPComponent(config *config.Config) (GitHTTPComponent, error) {
+ c := &gitHTTPComponentImpl{}
+ c.config = config
+ var err error
+ c.gitServer, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.s3Client, err = s3.NewMinio(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to init s3 client for code,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.lfsMetaObjectStore = database.NewLfsMetaObjectStore()
+ c.repoStore = database.NewRepoStore()
+ c.lfsLockStore = database.NewLfsLockStore()
+ c.userStore = database.NewUserStore()
+ c.repoComponent, err = NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, err
+ }
+ return c, nil
+}
+
+func (c *gitHTTPComponentImpl) InfoRefs(ctx context.Context, req types.InfoRefsReq) (io.Reader, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ if req.Rpc == "git-receive-pack" {
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !allowed {
+ return nil, ErrForbidden
+ }
+ } else {
+ if repo.Private {
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !allowed {
+ return nil, ErrForbidden
+ }
+ }
+ }
+
+ reader, err := c.gitServer.InfoRefsResponse(ctx, gitserver.InfoRefsReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Rpc: req.Rpc,
+ RepoType: req.RepoType,
+ GitProtocol: req.GitProtocol,
+ })
+
+ return reader, err
+}
+
+func (c *gitHTTPComponentImpl) GitUploadPack(ctx context.Context, req types.GitUploadPackReq) error {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ if repo.Private {
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ return ErrUnauthorized
+ }
+ if !allowed {
+ return ErrForbidden
+ }
+ }
+ err = c.gitServer.UploadPack(ctx, gitserver.UploadPackReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Request: req.Request,
+ RepoType: req.RepoType,
+ GitProtocol: req.GitProtocol,
+ Writer: req.Writer,
+ })
+
+ return err
+}
+
+func (c *gitHTTPComponentImpl) GitReceivePack(ctx context.Context, req types.GitReceivePackReq) error {
+ _, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return ErrUnauthorized
+ }
+
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ return ErrUnauthorized
+ }
+ if !allowed {
+ return ErrForbidden
+ }
+ err = c.gitServer.ReceivePack(ctx, gitserver.ReceivePackReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Request: req.Request,
+ RepoType: req.RepoType,
+ GitProtocol: req.GitProtocol,
+ Writer: req.Writer,
+ UserId: user.ID,
+ Username: user.Username,
+ })
+
+ return err
+}
+
+func (c *gitHTTPComponentImpl) BuildObjectResponse(ctx context.Context, req types.BatchRequest, isUpload bool) (*types.BatchResponse, error) {
+ var (
+ respObjects []*types.ObjectResponse
+ exists bool
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ for _, obj := range req.Objects {
+ if !obj.Valid() {
+ respObjects = append(respObjects, c.buildObjectResponse(ctx, req, obj, false, false, &types.ObjectError{
+ Code: http.StatusUnprocessableEntity,
+ Message: "Oid or size are invalid",
+ }))
+ continue
+ }
+ objectKey := path.Join("lfs", obj.RelativePath())
+ _, err := c.s3Client.StatObject(ctx, c.config.S3.Bucket, objectKey, minio.StatObjectOptions{})
+ if err != nil {
+ slog.Error("failed to check if lfs file exists", slog.String("oid", objectKey), slog.Any("error", err))
+ exists = false
+ } else {
+ exists = true
+ }
+
+ lfsMetaObject, err := c.lfsMetaObjectStore.FindByOID(ctx, repo.ID, obj.Oid)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ slog.Error("failed to check if lfs file exists in database", slog.String("oid", objectKey), slog.Any("error", err))
+ return nil, err
+ }
+
+ if lfsMetaObject != nil && obj.Size != lfsMetaObject.Size {
+ respObjects = append(respObjects, c.buildObjectResponse(ctx, req, obj, false, false, &types.ObjectError{
+ Code: http.StatusUnprocessableEntity,
+ Message: fmt.Sprintf("Object %s is not %d bytes", obj.Oid, obj.Size),
+ }))
+ continue
+ }
+
+ var responseObject *types.ObjectResponse
+ if isUpload {
+ var err *types.ObjectError
+ // if !exists && setting.LFS.MaxFileSize > 0 && p.Size > setting.LFS.MaxFileSize {
+ // err = &types.ObjectError{
+ // Code: http.StatusUnprocessableEntity,
+ // Message: fmt.Sprintf("Size must be less than or equal to %d", setting.LFS.MaxFileSize),
+ // }
+ // }
+
+ if exists && lfsMetaObject == nil {
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("unable to check if user can wirte this repo", slog.String("lfs oid", obj.Oid), slog.Any("error", err))
+ return nil, ErrUnauthorized
+ }
+ if allowed {
+ _, err := c.lfsMetaObjectStore.Create(ctx, database.LfsMetaObject{
+ Oid: obj.Oid,
+ Size: obj.Size,
+ RepositoryID: repo.ID,
+ Existing: true,
+ })
+ if err != nil {
+ slog.Error("Unable to create LFS MetaObject [%s] for %s/%s. Error: %v", obj.Oid, req.Namespace, req.Name, err)
+ return nil, err
+ }
+ } else {
+ exists = false
+ }
+ }
+
+ responseObject = c.buildObjectResponse(ctx, req, obj, false, !exists, err)
+ } else {
+ var err *types.ObjectError
+ // if !exists || lfsMetaObject == nil {
+ // err = &types.ObjectError{
+ // Code: http.StatusNotFound,
+ // Message: http.StatusText(http.StatusNotFound),
+ // }
+ // }
+
+ responseObject = c.buildObjectResponse(ctx, req, obj, true, false, err)
+ }
+ respObjects = append(respObjects, responseObject)
+ }
+ respobj := &types.BatchResponse{Objects: respObjects}
+ return respobj, nil
+}
+
+func (c *gitHTTPComponentImpl) buildObjectResponse(ctx context.Context, req types.BatchRequest, pointer types.Pointer, download, upload bool, err *types.ObjectError) *types.ObjectResponse {
+ rep := &types.ObjectResponse{Pointer: pointer}
+ if err != nil {
+ rep.Error = err
+ } else {
+ rep.Actions = make(map[string]*types.Link)
+
+ header := make(map[string]string)
+
+ if len(req.Authorization) > 0 {
+ header["Authorization"] = req.Authorization
+ }
+
+ if download {
+ var link *types.Link
+ reqParams := make(url.Values)
+ objectKey := path.Join("lfs", pointer.RelativePath())
+ url, err := c.s3Client.PresignedGetObjectGeo(ctx, c.config.S3.Bucket, objectKey, ossFileExpire, reqParams)
+ if url != nil && err == nil {
+ delete(header, "Authorization")
+ link = &types.Link{Href: url.String(), Header: header}
+ }
+ if link == nil {
+ link = &types.Link{Href: c.buildDownloadLink(req, pointer), Header: header}
+ }
+ rep.Actions["download"] = link
+ }
+ if upload {
+ rep.Actions["upload"] = &types.Link{Href: c.buildUploadLink(req, pointer), Header: header}
+
+ verifyHeader := make(map[string]string)
+ for key, value := range header {
+ verifyHeader[key] = value
+ }
+
+ verifyHeader["Accept"] = types.LfsMediaType
+
+ rep.Actions["verify"] = &types.Link{Href: c.buildVerifyLink(req), Header: verifyHeader}
+ }
+ }
+ return rep
+}
+
+func (c *gitHTTPComponentImpl) LfsUpload(ctx context.Context, body io.ReadCloser, req types.UploadRequest) error {
+ var exists bool
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ pointer := types.Pointer{Oid: req.Oid, Size: req.Size}
+
+ if !pointer.Valid() {
+ slog.Error("invalid lfs oid", slog.String("oid", req.Oid))
+ return errors.New("invalid lfs oid")
+ }
+
+ objectKey := path.Join("lfs", pointer.RelativePath())
+ _, err = c.s3Client.StatObject(ctx, c.config.S3.Bucket, objectKey, minio.StatObjectOptions{})
+ if err != nil {
+ if os.IsNotExist(err) {
+ exists = false
+ }
+ slog.Error("failed to check if lfs file exists", slog.String("oid", objectKey), slog.Any("error", err))
+ exists = false
+ } else {
+ exists = true
+ }
+ uploadOrVerify := func() error {
+ if exists {
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("Unable to check if LFS MetaObject [%s] is allowed. Error: %v", pointer.Oid, err)
+ return err
+ }
+ if !allowed {
+ // The file exists but the user has no access to it.
+ // The upload gets verified by hashing and size comparison to prove access to it.
+ hash := sha256.New()
+ written, err := io.Copy(hash, body)
+ if err != nil {
+ slog.Error("Error creating hash. Error: %v", slog.Any("error", err))
+ return err
+ }
+
+ if written != pointer.Size {
+ return types.ErrSizeMismatch
+ }
+ if hex.EncodeToString(hash.Sum(nil)) != pointer.Oid {
+ return types.ErrHashMismatch
+ }
+ }
+ } else {
+ var (
+ uploadErr error
+ uploadInfo minio.UploadInfo
+ )
+ uploadInfo, uploadErr = c.s3Client.PutObject(
+ ctx,
+ c.config.S3.Bucket,
+ objectKey,
+ body,
+ req.Size,
+ minio.PutObjectOptions{
+ ContentType: "application/octet-stream",
+ SendContentMd5: true,
+ ConcurrentStreamParts: true,
+ NumThreads: 5,
+ })
+ if uploadErr != nil {
+ slog.Error("Error putting LFS MetaObject [%s] into content store. Error: %v", pointer.Oid, err)
+ }
+ if uploadInfo.Size != pointer.Size {
+ uploadErr = types.ErrSizeMismatch
+ }
+ if uploadErr != nil {
+ err := c.s3Client.RemoveObject(
+ ctx,
+ c.config.S3.Bucket,
+ objectKey,
+ minio.RemoveObjectOptions{},
+ )
+ if err != nil {
+ slog.Error("Cleaning the LFS OID[%s] failed: %v", pointer.Oid, err)
+ }
+ }
+ }
+ _, err := c.lfsMetaObjectStore.Create(ctx, database.LfsMetaObject{
+ Oid: pointer.Oid,
+ Size: pointer.Size,
+ RepositoryID: repo.ID,
+ Existing: true,
+ })
+ return err
+ }
+ defer body.Close()
+ if err := uploadOrVerify(); err != nil {
+ if errors.Is(err, types.ErrSizeMismatch) || errors.Is(err, types.ErrHashMismatch) {
+ slog.Error("Upload does not match LFS MetaObject [%s]. Error: %v", pointer.Oid, err)
+ } else {
+ slog.Error("Error whilst uploadOrVerify LFS OID[%s]: %v", pointer.Oid, err)
+ }
+ if err = c.lfsMetaObjectStore.RemoveByOid(ctx, pointer.Oid, repo.ID); err != nil {
+ slog.Error("Error whilst removing MetaObject for LFS OID[%s]: %v", pointer.Oid, err)
+ }
+ return err
+ }
+
+ return nil
+}
+
+func (c *gitHTTPComponentImpl) LfsVerify(ctx context.Context, req types.VerifyRequest, p types.Pointer) error {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ // _, err = c.lfsMetaObjectStore.FindByOID(ctx, repo.ID, p.Oid)
+ // if err != nil {
+ // return fmt.Errorf("failed to find lfs meta object, error: %w", err)
+ // }
+ objectKey := path.Join("lfs", p.RelativePath())
+ fileInfo, err := c.s3Client.StatObject(ctx, c.config.S3.Bucket, objectKey, minio.StatObjectOptions{})
+ if err != nil {
+ slog.Error("failed to stat object in s3", slog.Any("error", err))
+ return fmt.Errorf("failed to stat object in s3, error: %w", err)
+ }
+
+ if fileInfo.Size != p.Size {
+ return types.ErrSizeMismatch
+ }
+
+ _, err = c.lfsMetaObjectStore.Create(ctx, database.LfsMetaObject{
+ Oid: p.Oid,
+ Size: p.Size,
+ RepositoryID: repo.ID,
+ Existing: true,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to create lfs meta object in database: %w", err)
+ }
+
+ return nil
+}
+
+func (c *gitHTTPComponentImpl) CreateLock(ctx context.Context, req types.LfsLockReq) (*database.LfsLock, error) {
+ var (
+ lock *database.LfsLock
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("Unable to check user write access:", slog.Any("error", err))
+ return nil, err
+ }
+
+ if !allowed {
+ return nil, ErrUnauthorized
+ }
+ lfsLock := database.LfsLock{
+ Path: req.Path,
+ UserID: user.ID,
+ RepositoryID: repo.ID,
+ }
+
+ lock, err = c.lfsLockStore.FindByPath(ctx, lfsLock.RepositoryID, lfsLock.Path)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ lock, err = c.lfsLockStore.Create(ctx, lfsLock)
+ if err != nil {
+ return nil, ErrAlreadyExists
+ }
+ return lock, nil
+ }
+ return lock, fmt.Errorf("failed to find lfs lock, error: %w", err)
+ }
+
+ return lock, ErrAlreadyExists
+}
+
+func (c *gitHTTPComponentImpl) ListLocks(ctx context.Context, req types.ListLFSLockReq) (*types.LFSLockList, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ _, err = c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("Unable to check user write access:", slog.Any("error", err))
+ return nil, err
+ }
+
+ if !allowed {
+ return nil, ErrUnauthorized
+ }
+
+ if req.ID != 0 {
+ l, err := c.lfsLockStore.FindByID(ctx, req.ID)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return buildLFSLockList([]database.LfsLock{}), nil
+ }
+ return buildLFSLockList([]database.LfsLock{}), err
+ }
+ if l.RepositoryID != repo.ID {
+ return buildLFSLockList([]database.LfsLock{}), nil
+ }
+ return buildLFSLockList([]database.LfsLock{*l}), nil
+ }
+
+ if req.Path != "" {
+ l, err := c.lfsLockStore.FindByPath(ctx, repo.ID, req.Path)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return buildLFSLockList([]database.LfsLock{}), nil
+ }
+ return buildLFSLockList([]database.LfsLock{}), err
+ }
+ return buildLFSLockList([]database.LfsLock{*l}), nil
+ }
+
+ locks, err := c.lfsLockStore.FindByRepoID(ctx, repo.ID, req.Cursor, req.Limit)
+ if err != nil {
+ return buildLFSLockList([]database.LfsLock{}), err
+ }
+
+ next := ""
+ if req.Limit > 0 && len(locks) == req.Limit {
+ next = strconv.Itoa(req.Cursor + 1)
+ }
+ res := buildLFSLockList(locks)
+ res.Next = next
+
+ return res, nil
+}
+
+func (c *gitHTTPComponentImpl) UnLock(ctx context.Context, req types.UnlockLFSReq) (*database.LfsLock, error) {
+ var (
+ lock *database.LfsLock
+ err error
+ )
+ _, err = c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("Unable to check user write access:", slog.Any("error", err))
+ return nil, err
+ }
+
+ if !allowed {
+ return nil, ErrUnauthorized
+ }
+
+ lock, err = c.lfsLockStore.FindByID(ctx, req.ID)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, ErrNotFound
+ }
+ return nil, err
+ }
+
+ if !req.Force && lock.UserID != user.ID {
+ return nil, ErrPermissionDenied
+ }
+ err = c.lfsLockStore.RemoveByID(ctx, req.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ return lock, nil
+}
+
+func (c *gitHTTPComponentImpl) VerifyLock(ctx context.Context, req types.VerifyLFSLockReq) (*types.LFSLockListVerify, error) {
+ var (
+ ourLocks []*types.LFSLock
+ theirLocks []*types.LFSLock
+ res types.LFSLockListVerify
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("Unable to check user write access:", slog.Any("error", err))
+ return nil, err
+ }
+
+ if !allowed {
+ return nil, ErrUnauthorized
+ }
+
+ locks, err := c.lfsLockStore.FindByRepoID(ctx, repo.ID, req.Cursor, req.Limit)
+ if err != nil {
+ return &types.LFSLockListVerify{}, err
+ }
+
+ next := ""
+ if req.Limit > 0 && len(locks) == req.Limit {
+ next = strconv.Itoa(req.Cursor + 1)
+ }
+ res.Next = next
+
+ for _, l := range locks {
+ lo := &types.LFSLock{
+ ID: strconv.FormatInt(l.ID, 10),
+ Path: l.Path,
+ LockedAt: l.CreatedAt,
+ Owner: &types.LFSLockOwner{
+ Name: l.User.Username,
+ },
+ }
+ if l.UserID == user.ID {
+ ourLocks = append(ourLocks, lo)
+ } else {
+ theirLocks = append(theirLocks, lo)
+ }
+ }
+ res.Ours = ourLocks
+ res.Theirs = theirLocks
+
+ return &res, nil
+}
+
+func (c *gitHTTPComponentImpl) LfsDownload(ctx context.Context, req types.DownloadRequest) (*url.URL, error) {
+ pointer := types.Pointer{Oid: req.Oid}
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check allowed, error: %w", err)
+ }
+
+ if !allowed {
+ return nil, errors.New("you have no permission to access this repo")
+ }
+
+ _, err = c.lfsMetaObjectStore.FindByOID(ctx, repo.ID, pointer.Oid)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find lfs meta object, error: %w", err)
+ }
+ objectKey := path.Join("lfs", pointer.RelativePath())
+
+ reqParams := make(url.Values)
+ if req.SaveAs != "" {
+ // allow rename when download through content-disposition header
+ reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", req.SaveAs))
+ }
+ signedUrl, err := c.s3Client.PresignedGetObjectGeo(ctx, c.config.S3.Bucket, objectKey, ossFileExpire, reqParams)
+ if err != nil {
+ return nil, err
+ }
+ return signedUrl, nil
+}
+
+func (c *gitHTTPComponentImpl) buildDownloadLink(req types.BatchRequest, pointer types.Pointer) string {
+ return c.config.APIServer.PublicDomain + "/" + path.Join(fmt.Sprintf("%ss", req.RepoType), url.PathEscape(req.Namespace), url.PathEscape(req.Name+".git"), "info/lfs/objects", url.PathEscape(pointer.Oid))
+}
+
+// func (c *gitHTTPComponentImpl) buildUploadLink(req types.BatchRequest, pointer types.Pointer) string {
+// return c.config.APIServer.PublicDomain + "/" + path.Join(fmt.Sprintf("%ss", req.RepoType), url.PathEscape(req.Namespace), url.PathEscape(req.Name+".git"), "info/lfs/objects", url.PathEscape(pointer.Oid), strconv.FormatInt(pointer.Size, 10))
+// }
+
+func (c *gitHTTPComponentImpl) buildUploadLink(req types.BatchRequest, pointer types.Pointer) string {
+ objectKey := path.Join("lfs", pointer.RelativePath())
+ u, err := c.s3Client.PresignedPutObject(context.Background(), c.config.S3.Bucket, objectKey, time.Hour*24)
+ if err != nil {
+ return c.config.APIServer.PublicDomain + "/" + path.Join(fmt.Sprintf("%ss", req.RepoType), url.PathEscape(req.Namespace), url.PathEscape(req.Name+".git"), "info/lfs/objects", url.PathEscape(pointer.Oid), strconv.FormatInt(pointer.Size, 10))
+ }
+ return u.String()
+}
+
+func (c *gitHTTPComponentImpl) buildVerifyLink(req types.BatchRequest) string {
+ return c.config.APIServer.PublicDomain + "/" + path.Join(fmt.Sprintf("%ss", req.RepoType), url.PathEscape(req.Namespace), url.PathEscape(req.Name+".git"), "info/lfs/verify")
+}
+
+func buildLFSLockList(lfsLocks []database.LfsLock) *types.LFSLockList {
+ if len(lfsLocks) == 0 {
+ return &types.LFSLockList{
+ Locks: []*types.LFSLock{},
+ }
+ }
+
+ var locks []*types.LFSLock
+ for _, l := range lfsLocks {
+ locks = append(locks, &types.LFSLock{
+ ID: strconv.FormatInt(l.ID, 10),
+ Path: l.Path,
+ LockedAt: l.CreatedAt,
+ Owner: &types.LFSLockOwner{
+ Name: l.User.Username,
+ },
+ })
+ }
+ return &types.LFSLockList{
+ Locks: locks,
+ }
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type HFDatasetComponent interface {
+ GetPathsInfo(ctx context.Context, req types.PathReq) ([]types.HFDSPathInfo, error)
+ GetDatasetTree(ctx context.Context, req types.PathReq) ([]types.HFDSPathInfo, error)
+}
+
+func NewHFDatasetComponent(config *config.Config) (HFDatasetComponent, error) {
+ c := &hFDatasetComponentImpl{}
+ c.ts = database.NewTagStore()
+ c.ds = database.NewDatasetStore()
+ c.rs = database.NewRepoStore()
+ var err error
+ c.repoComponentImpl, err = NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, err
+ }
+ return c, nil
+}
+
+type hFDatasetComponentImpl struct {
+ *repoComponentImpl
+ ts database.TagStore
+ ds database.DatasetStore
+ rs database.RepoStore
+}
+
+func convertFilePathFromRoute(path string) string {
+ return strings.TrimLeft(path, "/")
+}
+
+func (h *hFDatasetComponentImpl) GetPathsInfo(ctx context.Context, req types.PathReq) ([]types.HFDSPathInfo, error) {
+ ds, err := h.ds.FindByPath(ctx, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset, error: %w", err)
+ }
+
+ allow, err := h.AllowReadAccessRepo(ctx, ds.Repository, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check dataset permission, error: %w", err)
+ }
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: convertFilePathFromRoute(req.Path),
+ RepoType: types.DatasetRepo,
+ }
+ file, _ := h.git.GetRepoFileContents(ctx, getRepoFileTree)
+ if file == nil {
+ return []types.HFDSPathInfo{}, nil
+ }
+ slog.Debug("get file info", slog.Any("req", req), slog.Any("file", file))
+
+ paths := []types.HFDSPathInfo{
+ {
+ Type: "file",
+ Path: file.Path,
+ Size: file.Size,
+ OID: file.LastCommitSHA,
+ },
+ }
+
+ return paths, nil
+}
+
+func (h *hFDatasetComponentImpl) GetDatasetTree(ctx context.Context, req types.PathReq) ([]types.HFDSPathInfo, error) {
+ ds, err := h.ds.FindByPath(ctx, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find dataset tree, error: %w", err)
+ }
+
+ allow, err := h.AllowReadAccessRepo(ctx, ds.Repository, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check dataset permission, error: %w", err)
+ }
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ var treeFiles []types.HFDSPathInfo
+
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Path: req.Path,
+ RepoType: types.DatasetRepo,
+ }
+ tree, err := h.git.GetRepoFileTree(ctx, getRepoFileTree)
+ if err != nil {
+ slog.Warn("failed to get repo file tree", slog.Any("getRepoFileTree", getRepoFileTree), slog.String("error", err.Error()))
+ return []types.HFDSPathInfo{}, nil
+ }
+ slog.Debug("get tree", slog.Any("tree", tree))
+
+ for _, item := range tree {
+ treeFiles = append(treeFiles, types.HFDSPathInfo{Type: item.Type, OID: item.LastCommitSHA, Size: item.Size, Path: item.Path})
+ }
+ return treeFiles, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "path/filepath"
+ "strconv"
+
+ pb "gitlab.com/gitlab-org/gitaly/v16/proto/go/gitalypb"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/gitserver/gitaly"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+type internalComponentImpl struct {
+ config *config.Config
+ sshKeyStore database.SSHKeyStore
+ repoStore database.RepoStore
+ tokenStore database.AccessTokenStore
+ namespaceStore database.NamespaceStore
+ repoComponent RepoComponent
+ gitServer gitserver.GitServer
+}
+
+type InternalComponent interface {
+ Allowed(ctx context.Context) (bool, error)
+ SSHAllowed(ctx context.Context, req types.SSHAllowedReq) (*types.SSHAllowedResp, error)
+ GetAuthorizedKeys(ctx context.Context, key string) (*database.SSHKey, error)
+ GetCommitDiff(ctx context.Context, req types.GetDiffBetweenTwoCommitsReq) (*types.GiteaCallbackPushReq, error)
+ LfsAuthenticate(ctx context.Context, req types.LfsAuthenticateReq) (*types.LfsAuthenticateResp, error)
+}
+
+func NewInternalComponent(config *config.Config) (InternalComponent, error) {
+ var err error
+ c := &internalComponentImpl{}
+ c.config = config
+ c.sshKeyStore = database.NewSSHKeyStore()
+ c.repoStore = database.NewRepoStore()
+ c.repoComponent, err = NewRepoComponentImpl(config)
+ c.tokenStore = database.NewAccessTokenStore()
+ c.namespaceStore = database.NewNamespaceStore()
+ if err != nil {
+ return nil, err
+ }
+ git, err := git.NewGitServer(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server: %w", err)
+ }
+ c.gitServer = git
+ return c, nil
+}
+
+func (c *internalComponentImpl) Allowed(ctx context.Context) (bool, error) {
+ return true, nil
+}
+
+func (c *internalComponentImpl) SSHAllowed(ctx context.Context, req types.SSHAllowedReq) (*types.SSHAllowedResp, error) {
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find namespace %s: %v", req.Namespace, err)
+ }
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, err: %v", err)
+ }
+ if repo == nil {
+ return nil, errors.New("repo not found")
+ }
+ keyId, err := strconv.ParseInt(req.KeyID, 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse key ID, err: %v", err)
+ }
+ sshKey, err := c.sshKeyStore.FindByID(ctx, keyId)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find ssh key by id, err: %v", err)
+ }
+ if req.Action == "git-receive-pack" {
+ allowed, err := c.repoComponent.AllowWriteAccess(ctx, req.RepoType, req.Namespace, req.Name, sshKey.User.Username)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !allowed {
+ return nil, ErrForbidden
+ }
+ } else if req.Action == "git-upload-pack" {
+ if repo.Private {
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, sshKey.User.Username)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !allowed {
+ return nil, ErrForbidden
+ }
+ }
+ }
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+
+ return &types.SSHAllowedResp{
+ Success: true,
+ Message: "allowed",
+ Repo: req.Repo,
+ UserID: strconv.FormatInt(sshKey.User.ID, 10),
+ KeyType: "ssh",
+ KeyID: int(sshKey.ID),
+ ProjectID: int(repo.ID),
+ RootNamespaceID: int(namespace.ID),
+ GitConfigOptions: []string{"uploadpack.allowFilter=true", "uploadpack.allowAnySHA1InWant=true"},
+ Gitaly: types.Gitaly{
+ Repo: pb.Repository{
+ StorageName: c.config.GitalyServer.Storge,
+ RelativePath: gitaly.BuildRelativePath(repoType, req.Namespace, req.Name),
+ GlRepository: filepath.Join(repoType, req.Namespace, req.Name),
+ },
+ Address: c.config.GitalyServer.Address,
+ Token: c.config.GitalyServer.Token,
+ },
+ StatusCode: 200,
+ }, nil
+}
+
+func (c *internalComponentImpl) GetAuthorizedKeys(ctx context.Context, key string) (*database.SSHKey, error) {
+ fingerprint, err := common.CalculateAuthorizedSSHKeyFingerprint(key)
+ if err != nil {
+ return nil, fmt.Errorf("failed to calculate authorized keys fingerprint, error: %v", err)
+ }
+ sshKey, err := c.sshKeyStore.FindByFingerpringSHA256(ctx, fingerprint)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get authorized keys, error: %v", err)
+ }
+ return sshKey, nil
+}
+
+func (c *internalComponentImpl) GetCommitDiff(ctx context.Context, req types.GetDiffBetweenTwoCommitsReq) (*types.GiteaCallbackPushReq, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, err: %v", err)
+ }
+ if repo == nil {
+ return nil, errors.New("repo not found")
+ }
+ diffs, err := c.gitServer.GetDiffBetweenTwoCommits(ctx, gitserver.GetDiffBetweenTwoCommitsReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ RepoType: req.RepoType,
+ Ref: req.Ref,
+ LeftCommitId: req.LeftCommitId,
+ RightCommitId: req.RightCommitId,
+ Private: repo.Private,
+ })
+ if err != nil {
+ return nil, err
+ }
+ return diffs, nil
+}
+
+func (c *internalComponentImpl) LfsAuthenticate(ctx context.Context, req types.LfsAuthenticateReq) (*types.LfsAuthenticateResp, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, err: %v", err)
+ }
+ if repo == nil {
+ return nil, errors.New("repo not found")
+ }
+ keyId, err := strconv.ParseInt(req.KeyID, 10, 64)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse key ID, err: %v", err)
+ }
+ sshKey, err := c.sshKeyStore.FindByID(ctx, keyId)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find ssh key by id, err: %v", err)
+ }
+ if repo.Private {
+ allowed, err := c.repoComponent.AllowReadAccess(ctx, req.RepoType, req.Namespace, req.Name, sshKey.User.Username)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !allowed {
+ return nil, ErrForbidden
+ }
+ }
+ token, err := c.tokenStore.GetUserGitToken(ctx, sshKey.User.Username)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find git token by username, err: %v", err)
+ }
+ repoType := fmt.Sprintf("%ss", string(req.RepoType))
+ return &types.LfsAuthenticateResp{
+ Username: sshKey.User.Username,
+ LfsToken: token.Token,
+ RepoPath: c.config.APIServer.PublicDomain + "/" + filepath.Join(repoType, req.Namespace, req.Name+".git"),
+ }, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/rsa"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type LicenseComponent interface {
+ ListLicense(ctx context.Context, req types.QueryLicenseReq) ([]database.License, int, error)
+ CreateLicense(ctx context.Context, req *types.CreateLicenseReq) (string, error)
+ ImportLicense(ctx context.Context, req types.ImportLicenseReq) error
+ GetLicenseByID(ctx context.Context, req types.GetLicenseReq) (*database.License, string, error)
+ UpdateLicense(ctx context.Context, id int64, req *types.UpdateLicenseReq) (*database.License, error)
+ GetLicenseStatus(ctx context.Context) (*types.LicenseStatusResp, error)
+ DeleteLicenseByID(ctx context.Context, id int64, currentUser string) error
+ VerifyLicense(ctx context.Context, req types.ImportLicenseReq) (*types.RSAPayload, error)
+}
+
+type licenseComponentImpl struct {
+ publicKeyFile string
+ privateKeyFile string
+ user database.UserStore
+ ls database.LicenseStore
+ kr rsa.KeysReader
+}
+
+func NewLicenseComponent(config *config.Config) (LicenseComponent, error) {
+ lc := &licenseComponentImpl{
+ publicKeyFile: config.License.PublicKeyFile,
+ privateKeyFile: config.License.PrivateKeyFile,
+ user: database.NewUserStore(),
+ ls: database.NewLicenseStore(),
+ kr: rsa.NewKeysReader(),
+ }
+ return lc, nil
+}
+
+func NewTestLicenseComponent(
+ config *config.Config,
+ user database.UserStore,
+ ls database.LicenseStore,
+ kr rsa.KeysReader,
+) (LicenseComponent, error) {
+ lc := &licenseComponentImpl{
+ publicKeyFile: config.License.PublicKeyFile,
+ privateKeyFile: config.License.PrivateKeyFile,
+ user: user,
+ ls: ls,
+ kr: kr,
+ }
+ return lc, nil
+}
+
+func (c *licenseComponentImpl) ListLicense(ctx context.Context, req types.QueryLicenseReq) ([]database.License, int, error) {
+ user, err := c.user.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, 0, fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, 0, fmt.Errorf("user does not have permission to list licenses")
+ }
+ licenses, total, err := c.ls.List(ctx, req)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to list license: %v", err)
+ }
+
+ activeLic, err := c.ls.GetLatestActive(ctx)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, 0, fmt.Errorf("failed to get latest active license error: %w", err)
+ }
+
+ var decodedLicenses []database.License
+ for _, lic := range licenses {
+ if activeLic != nil && activeLic.ID == lic.ID {
+ lic.Status = types.LicenseStatusActive
+ } else if lic.ExpireTime.Before(time.Now()) {
+ lic.Status = types.LicenseStatusExpired
+ } else {
+ lic.Status = types.LicenseStatusInactive
+ }
+ decodedLicenses = append(decodedLicenses, lic)
+ }
+ return decodedLicenses, total, nil
+}
+
+func (c *licenseComponentImpl) CreateLicense(ctx context.Context, req *types.CreateLicenseReq) (string, error) {
+ user, err := c.user.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return "", fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return "", fmt.Errorf("user does not have permission to create license")
+ }
+
+ licenseKey := req.Key
+ if len(licenseKey) == 0 {
+ licenseKey = uuid.New().String()
+ }
+
+ input := database.License{
+ Key: licenseKey,
+ Company: req.Company,
+ Email: req.Email,
+ Product: req.Product,
+ Edition: req.Edition,
+ Version: req.Version,
+ MaxUser: req.MaxUser,
+ StartTime: req.StartTime,
+ ExpireTime: req.ExpireTime,
+ Extra: req.Extra,
+ Remark: req.Remark,
+ UserUUID: user.UUID,
+ }
+
+ err = c.ls.Create(ctx, input)
+ if err != nil {
+ return "", fmt.Errorf("failed to save license info, error: %w", err)
+ }
+
+ payload := types.RSAPayload{
+ Key: licenseKey,
+ DataBody: req.DataBody,
+ }
+ encodedLicense, err := rsa.GenerateData(c.kr, c.privateKeyFile, payload)
+ if err != nil {
+ return "", fmt.Errorf("can not generate license with error: %w", err)
+ }
+ return encodedLicense, nil
+}
+
+func (c *licenseComponentImpl) ImportLicense(ctx context.Context, req types.ImportLicenseReq) error {
+ user, err := c.user.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return fmt.Errorf("user does not have permission to import license")
+ }
+
+ licenseRes, err := rsa.VerifyData(c.kr, c.publicKeyFile, req.Data)
+ if err != nil {
+ return fmt.Errorf("fail to verify license with error: %w", err)
+ }
+ slog.Debug("Verify License", slog.Any("licenseRes", licenseRes))
+
+ input := database.License{
+ Key: licenseRes.Key,
+ Company: licenseRes.Company,
+ Email: licenseRes.Email,
+ Product: licenseRes.Product,
+ Edition: licenseRes.Edition,
+ Version: licenseRes.Version,
+ MaxUser: licenseRes.MaxUser,
+ StartTime: licenseRes.StartTime,
+ ExpireTime: licenseRes.ExpireTime,
+ Extra: licenseRes.Extra,
+ UserUUID: user.UUID,
+ Remark: "-",
+ }
+
+ err = c.ls.Create(ctx, input)
+ if err != nil {
+ return fmt.Errorf("failed to import license, error: %w", err)
+ }
+ return nil
+}
+
+func (c *licenseComponentImpl) GetLicenseByID(ctx context.Context, req types.GetLicenseReq) (*database.License, string, error) {
+ user, err := c.user.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, "", fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, "", fmt.Errorf("user does not have permission to get license")
+ }
+ license, err := c.ls.GetByID(ctx, req.ID)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to get license by id %d, error: %w", req.ID, err)
+ }
+ var fileContent string
+ if req.CreateFile {
+ payload := types.RSAPayload{
+ Key: license.Key,
+ DataBody: types.DataBody{
+ Company: license.Company,
+ Email: license.Email,
+ Product: license.Product,
+ Edition: license.Edition,
+ Version: license.Version,
+ MaxUser: license.MaxUser,
+ StartTime: license.StartTime,
+ ExpireTime: license.ExpireTime,
+ Extra: license.Extra,
+ },
+ }
+
+ fileContent, err = rsa.GenerateData(c.kr, c.privateKeyFile, payload)
+ if err != nil {
+ return nil, "", fmt.Errorf("can not generate license with error: %w", err)
+ }
+ }
+ return license, fileContent, nil
+}
+
+func (c *licenseComponentImpl) UpdateLicense(ctx context.Context, id int64, req *types.UpdateLicenseReq) (*database.License, error) {
+ user, err := c.user.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, fmt.Errorf("user does not have permission to update license")
+ }
+
+ input, err := c.ls.GetByID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get license by id %d, error: %w", id, err)
+ }
+
+ if req.Company != nil {
+ input.Company = *req.Company
+ }
+ if req.Email != nil {
+ input.Email = *req.Email
+ }
+ if req.Product != nil {
+ input.Product = *req.Product
+ }
+ if req.Edition != nil {
+ input.Edition = *req.Edition
+ }
+ if req.Version != nil {
+ input.Version = *req.Version
+ }
+ if req.MaxUser != nil {
+ input.MaxUser = *req.MaxUser
+ }
+ if req.StartTime != nil {
+ input.StartTime = *req.StartTime
+ }
+ if req.ExpireTime != nil {
+ input.ExpireTime = *req.ExpireTime
+ }
+ if req.Extra != nil {
+ input.Extra = *req.Extra
+ }
+ if req.Remark != nil {
+ input.Remark = *req.Remark
+ }
+
+ err = c.ls.Update(ctx, *input)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update license, error: %w", err)
+ }
+ return input, nil
+}
+
+func (c *licenseComponentImpl) GetLicenseStatus(ctx context.Context) (*types.LicenseStatusResp, error) {
+ license, err := c.ls.GetLatestActive(ctx)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to get latest active license error: %w", err)
+ }
+ total, err := c.user.CountUsers(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to count users error: %w", err)
+ }
+ var status types.LicenseStatusResp
+ status.ID = license.ID
+ status.Key = license.Key
+ status.Company = license.Company
+ status.Email = license.Email
+ status.Product = license.Product
+ status.Edition = license.Edition
+ status.Version = license.Version
+ status.MaxUser = license.MaxUser
+ status.StartTime = license.StartTime
+ status.ExpireTime = license.ExpireTime
+ status.Extra = license.Extra
+ status.Users = total
+
+ return &status, nil
+}
+
+func (c *licenseComponentImpl) DeleteLicenseByID(ctx context.Context, id int64, currentUser string) error {
+ user, err := c.user.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return fmt.Errorf("user does not have permission to delete license")
+ }
+
+ err = c.ls.Delete(ctx, database.License{ID: id})
+ if err != nil {
+ return fmt.Errorf("failed to delete license by id %d error: %w", id, err)
+ }
+
+ return nil
+}
+
+func (c *licenseComponentImpl) VerifyLicense(ctx context.Context, req types.ImportLicenseReq) (*types.RSAPayload, error) {
+ user, err := c.user.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, fmt.Errorf("user does not have permission to import license")
+ }
+
+ licenseRes, err := rsa.VerifyData(c.kr, c.publicKeyFile, req.Data)
+ if err != nil {
+ return nil, fmt.Errorf("fail to verify license with error: %w", err)
+ }
+ slog.Debug("Verify License", slog.Any("licenseRes", licenseRes))
+
+ input := types.RSAPayload{
+ Key: licenseRes.Key,
+ DataBody: types.DataBody{
+ Company: licenseRes.Company,
+ Email: licenseRes.Email,
+ Product: licenseRes.Product,
+ Edition: licenseRes.Edition,
+ Version: licenseRes.Version,
+ MaxUser: licenseRes.MaxUser,
+ StartTime: licenseRes.StartTime,
+ ExpireTime: licenseRes.ExpireTime,
+ Extra: licenseRes.Extra,
+ },
+ }
+ return &input, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type ListComponent interface {
+ ListModelsByPath(ctx context.Context, req *types.ListByPathReq) ([]*types.ModelResp, error)
+ ListDatasetsByPath(ctx context.Context, req *types.ListByPathReq) ([]*types.DatasetResp, error)
+}
+
+func NewListComponent(config *config.Config) (ListComponent, error) {
+ c := &listComponentImpl{}
+ c.ds = database.NewDatasetStore()
+ c.ms = database.NewModelStore()
+ c.ss = database.NewSpaceStore()
+ return c, nil
+}
+
+type listComponentImpl struct {
+ ms database.ModelStore
+ ds database.DatasetStore
+ ss database.SpaceStore
+}
+
+func (c *listComponentImpl) ListModelsByPath(ctx context.Context, req *types.ListByPathReq) ([]*types.ModelResp, error) {
+ var modelResp []*types.ModelResp
+
+ models, err := c.ms.ListByPath(ctx, req.Paths)
+ if err != nil {
+ slog.Error("error listing models by path: %v", slog.Any("error", err), slog.Any("paths", req.Paths))
+ return nil, err
+ }
+ for _, model := range models {
+ var tags []types.RepoTag
+ for _, tag := range model.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ modelResp = append(modelResp, &types.ModelResp{
+ Name: model.Repository.Name,
+ Path: model.Repository.Path,
+ Downloads: model.Repository.DownloadCount,
+ UpdatedAt: model.UpdatedAt,
+ Private: model.Repository.Private,
+ Nickname: model.Repository.Nickname,
+ Description: model.Repository.Description,
+ Tags: tags,
+ })
+ }
+
+ return modelResp, nil
+}
+
+func (c *listComponentImpl) ListDatasetsByPath(ctx context.Context, req *types.ListByPathReq) ([]*types.DatasetResp, error) {
+ var datasetResp []*types.DatasetResp
+
+ datasets, err := c.ds.ListByPath(ctx, req.Paths)
+ if err != nil {
+ slog.Error("error listing datasets by path: %v", slog.Any("error", err), slog.Any("paths", req.Paths))
+ return nil, err
+ }
+ for _, dataset := range datasets {
+ var tags []types.RepoTag
+ for _, tag := range dataset.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ datasetResp = append(datasetResp, &types.ModelResp{
+ Name: dataset.Repository.Name,
+ Path: dataset.Repository.Path,
+ Downloads: dataset.Repository.DownloadCount,
+ UpdatedAt: dataset.UpdatedAt,
+ Private: dataset.Repository.Private,
+ Nickname: dataset.Repository.Nickname,
+ Description: dataset.Repository.Description,
+ Tags: tags,
+ })
+ }
+ return datasetResp, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "path"
+ "strings"
+
+ "github.com/minio/minio-go/v7"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/mirrorserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/s3"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+type mirrorComponentImpl struct {
+ tokenStore database.GitServerAccessTokenStore
+ mirrorServer mirrorserver.MirrorServer
+ saas bool
+ repoComp RepoComponent
+ git gitserver.GitServer
+ s3Client s3.Client
+ lfsBucket string
+ modelStore database.ModelStore
+ datasetStore database.DatasetStore
+ codeStore database.CodeStore
+ repoStore database.RepoStore
+ mirrorStore database.MirrorStore
+ mirrorSourceStore database.MirrorSourceStore
+ namespaceStore database.NamespaceStore
+ lfsMetaObjectStore database.LfsMetaObjectStore
+ userStore database.UserStore
+ config *config.Config
+ mq queue.PriorityQueue
+}
+
+type MirrorComponent interface {
+ CreatePushMirrorForFinishedMirrorTask(ctx context.Context) error
+ // CreateMirrorRepo often called by the crawler server to create new repo which will then be mirrored from other sources
+ CreateMirrorRepo(ctx context.Context, req types.CreateMirrorRepoReq) (*database.Mirror, error)
+ CheckMirrorProgress(ctx context.Context) error
+ Repos(ctx context.Context, currentUser string, per, page int) ([]types.MirrorRepo, int, error)
+ Index(ctx context.Context, currentUser string, per, page int, search string) ([]types.Mirror, int, error)
+ Statistics(ctx context.Context, currentUser string) ([]types.MirrorStatusCount, error)
+}
+
+func NewMirrorComponent(config *config.Config) (MirrorComponent, error) {
+ var err error
+ c := &mirrorComponentImpl{}
+ if config.GitServer.Type == types.GitServerTypeGitea {
+ c.mirrorServer, err = git.NewMirrorServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git mirror server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ }
+ c.mq, err = queue.GetPriorityQueueInstance()
+ if err != nil {
+ return nil, fmt.Errorf("failed to get priority queue: %v", err)
+ }
+
+ c.repoComp, err = NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create repo component,error:%w", err)
+ }
+ c.git, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.s3Client, err = s3.NewMinio(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to init s3 client for code,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.lfsBucket = config.S3.Bucket
+ c.modelStore = database.NewModelStore()
+ c.datasetStore = database.NewDatasetStore()
+ c.codeStore = database.NewCodeStore()
+ c.repoStore = database.NewRepoStore()
+ c.mirrorStore = database.NewMirrorStore()
+ c.tokenStore = database.NewGitServerAccessTokenStore()
+ c.mirrorSourceStore = database.NewMirrorSourceStore()
+ c.namespaceStore = database.NewNamespaceStore()
+ c.lfsMetaObjectStore = database.NewLfsMetaObjectStore()
+ c.userStore = database.NewUserStore()
+ c.saas = config.Saas
+ c.config = config
+ return c, nil
+}
+
+func (c *mirrorComponentImpl) CreatePushMirrorForFinishedMirrorTask(ctx context.Context) error {
+ mirrors, err := c.mirrorStore.NoPushMirror(ctx)
+ if err != nil {
+ return fmt.Errorf("fail to find all mirrors, %w", err)
+ }
+
+ for _, mirror := range mirrors {
+ task, err := c.mirrorServer.GetMirrorTaskInfo(ctx, mirror.MirrorTaskID)
+ if err != nil {
+ slog.Error("fail to get mirror task info", slog.Int64("taskId", mirror.MirrorTaskID), slog.String("error", err.Error()))
+ return fmt.Errorf("fail to get mirror task info, %w", err)
+ }
+ if task.Status == mirrorserver.TaskStatusFinished {
+ err = c.mirrorServer.CreatePushMirror(ctx, mirrorserver.CreatePushMirrorReq{
+ Name: mirror.LocalRepoPath,
+ PushUrl: mirror.PushUrl,
+ Username: mirror.PushUsername,
+ AccessToken: mirror.PushAccessToken,
+ Interval: "8h",
+ })
+
+ if err != nil {
+ slog.Error("fail to create push mirror", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ continue
+ }
+ mirror.PushMirrorCreated = true
+ err = c.mirrorStore.Update(ctx, &mirror)
+ if err != nil {
+ slog.Error("fail to update mirror", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ continue
+ }
+ slog.Info("create push mirror successfully", slog.Int64("mirrorId", mirror.ID), slog.String("push_url", mirror.PushUrl))
+ }
+ }
+ return nil
+}
+
+// CreateMirrorRepo often called by the crawler server to create new repo which will then be mirrored from other sources
+func (c *mirrorComponentImpl) CreateMirrorRepo(ctx context.Context, req types.CreateMirrorRepoReq) (*database.Mirror, error) {
+ var username string
+ namespace := c.mapNamespaceAndName(req.SourceNamespace)
+ name := req.SourceName
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, errors.New("user does not have admin permission")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, namespace, name)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, fmt.Errorf("failed to check repo existence, error: %w", err)
+ }
+ if repo != nil {
+ name = fmt.Sprintf("%s_%s", req.SourceNamespace, req.SourceName)
+ repo, err = c.repoStore.FindByPath(ctx, req.RepoType, namespace, name)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, fmt.Errorf("failed to check repo existence, error: %w", err)
+ }
+ if repo != nil {
+ return nil, fmt.Errorf("repo already exists,repo type:%s, source namespace: %s, source name: %s, target namespace: %s, target name: %s",
+ req.RepoType, req.SourceNamespace, req.SourceName, namespace, name)
+ }
+ }
+
+ dbNamespace, err := c.namespaceStore.FindByPath(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("namespace does not exist, namespace: %s", namespace)
+ }
+ username = dbNamespace.User.Username
+
+ // create repo, create mirror repo
+ gitRepo, repo, err := c.repoComp.CreateRepo(ctx, types.CreateRepoReq{
+ Username: username,
+ Namespace: namespace,
+ Name: name,
+ Nickname: name,
+ //TODO: tranlate description automatically
+ Description: req.Description,
+ //only mirror public repository
+ Private: true,
+ License: req.License,
+ DefaultBranch: req.DefaultBranch,
+ RepoType: req.RepoType,
+ })
+
+ if err != nil {
+ return nil, fmt.Errorf("failed to create OpenCSG repo, error: %w", err)
+ }
+
+ if req.RepoType == types.ModelRepo {
+ dbModel := database.Model{
+ Repository: repo,
+ RepositoryID: repo.ID,
+ }
+
+ _, err := c.modelStore.Create(ctx, dbModel)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create model, error: %w", err)
+ }
+ } else if req.RepoType == types.DatasetRepo {
+ dbDataset := database.Dataset{
+ Repository: repo,
+ RepositoryID: repo.ID,
+ }
+
+ _, err := c.datasetStore.Create(ctx, dbDataset)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create dataset, error: %w", err)
+ }
+ } else if req.RepoType == types.CodeRepo {
+ dbCode := database.Code{
+ Repository: repo,
+ RepositoryID: repo.ID,
+ }
+
+ _, err := c.codeStore.Create(ctx, dbCode)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create code, error: %w", err)
+ }
+ }
+
+ pushAccessToken, err := c.tokenStore.FindByType(ctx, "git")
+ if err != nil {
+ return nil, fmt.Errorf("failed to find git access token, error: %w", err)
+ }
+ if len(pushAccessToken) == 0 {
+ return nil, fmt.Errorf("failed to find git access token, error: empty table")
+ }
+
+ mirrorSource, err := c.mirrorSourceStore.Get(ctx, req.MirrorSourceID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find mirror Source, error: %w", err)
+ }
+ var mirror database.Mirror
+ // mirror.Interval = req.Interval
+ mirror.SourceUrl = req.SourceGitCloneUrl
+ mirror.MirrorSourceID = req.MirrorSourceID
+ mirror.PushUrl = gitRepo.HttpCloneURL
+ mirror.Username = req.SourceNamespace
+ mirror.PushUsername = "root"
+ //TODO: get user git access token from db git access token
+ mirror.PushAccessToken = pushAccessToken[0].Token
+ mirror.RepositoryID = repo.ID
+ mirror.Repository = repo
+ mirror.LocalRepoPath = fmt.Sprintf("%s_%s_%s_%s", mirrorSource.SourceName, req.RepoType, req.SourceNamespace, req.SourceName)
+ mirror.SourceRepoPath = fmt.Sprintf("%s/%s", req.SourceNamespace, req.SourceName)
+ mirror.Priority = types.HighMirrorPriority
+
+ var taskId int64
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ taskId, err = c.mirrorServer.CreateMirrorRepo(ctx, mirrorserver.CreateMirrorRepoReq{
+ Namespace: "root",
+ Name: mirror.LocalRepoPath,
+ CloneUrl: mirror.SourceUrl,
+ // Username: req.SourceNamespace,
+ // AccessToken: mirror.AccessToken,
+ Private: false,
+ SyncLfs: req.SyncLfs,
+ })
+
+ if err != nil {
+ return nil, fmt.Errorf("failed to create push mirror in mirror server: %v", err)
+ }
+ }
+ mirror.MirrorTaskID = taskId
+
+ reqMirror, err := c.mirrorStore.Create(ctx, &mirror)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create mirror")
+ }
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ c.mq.PushRepoMirror(&queue.MirrorTask{
+ MirrorID: reqMirror.ID,
+ Priority: queue.PriorityMap[reqMirror.Priority],
+ })
+ reqMirror.Status = types.MirrorWaiting
+ err = c.mirrorStore.Update(ctx, reqMirror)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update mirror status: %v", err)
+ }
+ }
+
+ return reqMirror, nil
+
+}
+
+func (m *mirrorComponentImpl) mapNamespaceAndName(sourceNamespace string) string {
+ var namespace string
+ if ns, found := mirrorOrganizationMap[sourceNamespace]; found {
+ namespace = ns
+ } else {
+ //map all organization to AIWizards if not found
+ namespace = "AIWizards"
+ }
+
+ return namespace
+}
+
+func (c *mirrorComponentImpl) CheckMirrorProgress(ctx context.Context) error {
+ mirrors, err := c.mirrorStore.Unfinished(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to get unfinished mirrors: %v", err)
+ }
+ for _, mirror := range mirrors {
+ err := c.checkAndUpdateMirrorStatus(ctx, mirror)
+ if err != nil {
+ slog.Error("fail to check and update mirror status", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ }
+ }
+ return nil
+}
+
+var mirrorOrganizationMap = map[string]string{
+ "THUDM": "THUDM",
+ "baichuan-inc": "BaiChuanAI",
+ "IDEA-CCNL": "FengShenBang",
+ "internlm": "ShangHaiAILab",
+ "pleisto": "Pleisto",
+ "01-ai": "01AI",
+ "codefuse-ai": "codefuse-ai",
+ "WisdomShell": "WisdomShell",
+ "microsoft": "microsoft",
+ "Skywork": "Skywork",
+ "BAAI": "BAAI",
+ "deepseek-ai": "deepseek-ai",
+ "WizardLMTeam": "WizardLM",
+ "IEITYuan": "IEITYuan",
+ "Qwen": "Qwen",
+ "TencentARC": "TencentARC",
+ "OrionStarAI": "OrionStarAI",
+ "openbmb": "OpenBMB",
+ "netease-youdao": "Netease-youdao",
+ "ByteDance": "ByteDance",
+ "opencompass": "opencompass",
+}
+
+var mirrorStatusAndRepoSyncStatusMapping = map[types.MirrorTaskStatus]types.RepositorySyncStatus{
+ types.MirrorWaiting: types.SyncStatusPending,
+ types.MirrorRunning: types.SyncStatusInProgress,
+ types.MirrorFinished: types.SyncStatusCompleted,
+ types.MirrorFailed: types.SyncStatusFailed,
+ types.MirrorIncomplete: types.SyncStatusFailed,
+}
+
+func (c *mirrorComponentImpl) checkAndUpdateMirrorStatus(ctx context.Context, mirror database.Mirror) error {
+ var statusAndProgressFunc func(ctx context.Context, mirror database.Mirror) (types.MirrorResp, error)
+ if mirror.Repository == nil {
+ return nil
+ }
+ if c.saas {
+ statusAndProgressFunc = c.getMirrorStatusAndProgressSaas
+ } else {
+ statusAndProgressFunc = c.getMirrorStatusAndProgressOnPremise
+ }
+ mirrorResp, err := statusAndProgressFunc(ctx, mirror)
+ if err != nil {
+ slog.Error("fail to get mirror status and progress", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ }
+ mirror.Status = mirrorResp.TaskStatus
+ mirror.Progress = mirrorResp.Progress
+ mirror.LastMessage = mirrorResp.LastMessage
+ err = c.mirrorStore.Update(ctx, &mirror)
+ if err != nil {
+ slog.Error("fail to update mirror", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ return err
+ }
+ if mirror.Repository.HTTPCloneURL == "" {
+ namespace, name := mirror.Repository.NamespaceAndName()
+ repoRes, err := c.git.GetRepo(ctx, gitserver.GetRepoReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: mirror.Repository.RepositoryType,
+ })
+ if err != nil {
+ slog.Error("fail to get repo detail from git server")
+ } else {
+ mirror.Repository.HTTPCloneURL = common.PortalCloneUrl(repoRes.HttpCloneURL, mirror.Repository.RepositoryType, c.config.GitServer.URL, c.config.Frontend.URL)
+ mirror.Repository.SSHCloneURL = repoRes.SshCloneURL
+ mirror.Repository.DefaultBranch = repoRes.DefaultBranch
+ }
+ }
+ syncStatus := mirrorStatusAndRepoSyncStatusMapping[mirrorResp.TaskStatus]
+ mirror.Repository.SyncStatus = syncStatus
+ _, err = c.repoStore.UpdateRepo(ctx, *mirror.Repository)
+ if err != nil {
+ slog.Error("fail to update repo sync status", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ return err
+ }
+
+ return nil
+}
+
+func getAllFiles(namespace, repoName, folder string, repoType types.RepositoryType, ref string, gsTree func(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error)) ([]*types.File, error) {
+ var files []*types.File
+
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: repoName,
+ Ref: ref,
+ Path: folder,
+ RepoType: repoType,
+ }
+ gitFiles, err := gsTree(context.Background(), getRepoFileTree)
+ if err != nil {
+ return files, fmt.Errorf("failed to get repo file tree,%w", err)
+ }
+ for _, file := range gitFiles {
+ if file.Type == "dir" {
+ subFiles, err := getAllFiles(namespace, repoName, file.Path, repoType, ref, gsTree)
+ if err != nil {
+ return files, err
+ }
+ files = append(files, subFiles...)
+ } else {
+ files = append(files, file)
+ }
+ }
+ return files, nil
+}
+
+func (c *mirrorComponentImpl) getMirrorStatusAndProgressOnPremise(ctx context.Context, mirror database.Mirror) (types.MirrorResp, error) {
+ task, err := c.git.GetMirrorTaskInfo(ctx, mirror.MirrorTaskID)
+ if err != nil {
+ slog.Error("fail to get mirror task info", slog.Int64("taskId", mirror.MirrorTaskID), slog.String("error", err.Error()))
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: "",
+ Progress: 0,
+ }, fmt.Errorf("fail to get mirror task info, %w", err)
+ }
+ if task.Status == gitserver.TaskStatusQueued {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorWaiting,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, nil
+ } else if task.Status == gitserver.TaskStatusRunning {
+ progress, err := c.countMirrorProgress(ctx, mirror)
+ if err != nil {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorRunning,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, err
+ }
+ return types.MirrorResp{
+ TaskStatus: types.MirrorRunning,
+ LastMessage: task.Message,
+ Progress: progress,
+ }, nil
+ } else if task.Status == gitserver.TaskStatusFailed {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, nil
+ } else if task.Status == gitserver.TaskStatusFinished {
+ progress, err := c.countMirrorProgress(ctx, mirror)
+ if err != nil {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, err
+ }
+ if progress == 100 {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFinished,
+ LastMessage: task.Message,
+ Progress: progress,
+ }, nil
+ } else {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorIncomplete,
+ LastMessage: task.Message,
+ Progress: progress,
+ }, nil
+ }
+ } else {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: "",
+ Progress: 0,
+ }, nil
+ }
+}
+
+func (c *mirrorComponentImpl) getMirrorStatusAndProgressSaas(ctx context.Context, mirror database.Mirror) (types.MirrorResp, error) {
+ task, err := c.mirrorServer.GetMirrorTaskInfo(ctx, mirror.MirrorTaskID)
+ if err != nil {
+ slog.Error("fail to get mirror task info", slog.Int64("taskId", mirror.MirrorTaskID), slog.String("error", err.Error()))
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: "",
+ Progress: 0,
+ }, fmt.Errorf("fail to get mirror task info, %w", err)
+ }
+ if task.Status == mirrorserver.TaskStatusQueued {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorWaiting,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, nil
+ } else if task.Status == mirrorserver.TaskStatusRunning {
+ progress, err := c.countMirrorProgress(ctx, mirror)
+ if err != nil {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorRunning,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, err
+ }
+ return types.MirrorResp{
+ TaskStatus: types.MirrorRunning,
+ LastMessage: task.Message,
+ Progress: progress,
+ }, nil
+ } else if task.Status == mirrorserver.TaskStatusFailed {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, nil
+ } else if task.Status == mirrorserver.TaskStatusFinished {
+ progress, err := c.countMirrorProgress(ctx, mirror)
+ if err != nil {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: task.Message,
+ Progress: 0,
+ }, err
+ }
+ if progress == 100 {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFinished,
+ LastMessage: task.Message,
+ Progress: progress,
+ }, nil
+ } else {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorIncomplete,
+ LastMessage: task.Message,
+ Progress: progress,
+ }, nil
+ }
+ } else {
+ return types.MirrorResp{
+ TaskStatus: types.MirrorFailed,
+ LastMessage: "",
+ Progress: 0,
+ }, nil
+ }
+}
+
+func (c *mirrorComponentImpl) countMirrorProgress(ctx context.Context, mirror database.Mirror) (int8, error) {
+ var (
+ lfsFiles []*types.File
+ finishedFileCount int
+ )
+ namespaceAndName := strings.Split(mirror.Repository.Path, "/")
+ namespace := namespaceAndName[0]
+ name := namespaceAndName[1]
+ allFiles, err := getAllFiles(namespace, name, "", mirror.Repository.RepositoryType, "", c.git.GetRepoFileTree)
+
+ if err != nil {
+ slog.Error("fail to get all files of mirror repository", slog.Int64("mirrorId", mirror.ID), slog.String("namespace", namespace), slog.String("name", name), slog.String("error", err.Error()))
+ return 0, err
+ }
+ if len(allFiles) == 0 {
+ return 0, nil
+ }
+ for _, f := range allFiles {
+ if f.Lfs {
+ lfsFiles = append(lfsFiles, f)
+ }
+ }
+ if len(lfsFiles) == 0 {
+ return 100, nil
+ }
+ for _, f := range lfsFiles {
+ objectKey := f.LfsRelativePath
+ objectKey = path.Join("lfs", objectKey)
+ _, err := c.s3Client.StatObject(ctx, c.lfsBucket, objectKey, minio.GetObjectOptions{})
+ if err != nil {
+ if minio.ToErrorResponse(err).Code != "NoSuchKey" {
+ slog.Error("fail to check lfs file", slog.Int64("mirrorId", mirror.ID), slog.String("namespace", namespace), slog.String("name", name), slog.String("filename", f.Path), slog.String("error", err.Error()))
+ return 0, err
+ }
+ } else {
+ finishedFileCount += 1
+ }
+ }
+
+ progress := (finishedFileCount * 100) / len(lfsFiles)
+ return int8(progress), nil
+}
+
+func (c *mirrorComponentImpl) Repos(ctx context.Context, currentUser string, per, page int) ([]types.MirrorRepo, int, error) {
+ var mirrorRepos []types.MirrorRepo
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, 0, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, 0, errors.New("user does not have admin permission")
+ }
+ repos, total, err := c.repoStore.WithMirror(ctx, per, page)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get mirror repositories: %v", err)
+ }
+ for _, repo := range repos {
+ mirrorRepos = append(mirrorRepos, types.MirrorRepo{
+ Path: repo.Path,
+ SyncStatus: repo.SyncStatus,
+ RepoType: repo.RepositoryType,
+ Progress: repo.Mirror.Progress,
+ })
+ }
+ return mirrorRepos, total, nil
+}
+
+func (c *mirrorComponentImpl) Index(ctx context.Context, currentUser string, per, page int, search string) ([]types.Mirror, int, error) {
+ var mirrorsResp []types.Mirror
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, 0, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, 0, errors.New("user does not have admin permission")
+ }
+ mirrors, total, err := c.mirrorStore.IndexWithPagination(ctx, per, page, search)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get mirror mirrors: %v", err)
+ }
+ for _, mirror := range mirrors {
+ if mirror.Repository != nil {
+ mirrorsResp = append(mirrorsResp, types.Mirror{
+ SourceUrl: mirror.SourceUrl,
+ MirrorSource: types.MirrorSource{
+ SourceName: mirror.MirrorSource.SourceName,
+ },
+ Username: mirror.Username,
+ AccessToken: mirror.AccessToken,
+ PushUrl: mirror.PushUrl,
+ PushUsername: mirror.PushUsername,
+ PushAccessToken: mirror.PushAccessToken,
+ LastUpdatedAt: mirror.LastUpdatedAt,
+ SourceRepoPath: mirror.SourceRepoPath,
+ LocalRepoPath: fmt.Sprintf("%ss/%s", mirror.Repository.RepositoryType, mirror.Repository.Path),
+ LastMessage: mirror.LastMessage,
+ Status: mirror.Status,
+ Progress: mirror.Progress,
+ })
+ }
+ }
+ return mirrorsResp, total, nil
+}
+
+func (c *mirrorComponentImpl) Statistics(ctx context.Context, currentUser string) ([]types.MirrorStatusCount, error) {
+ var scs []types.MirrorStatusCount
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, errors.New("user does not have admin permission")
+ }
+ statusCounts, err := c.mirrorStore.StatusCount(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get mirror statistics: %v", err)
+ }
+
+ for _, statusCount := range statusCounts {
+ scs = append(scs, types.MirrorStatusCount{
+ Status: statusCount.Status,
+ Count: statusCount.Count,
+ })
+ }
+
+ return scs, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type mirrorSourceComponentImpl struct {
+ mirrorSourceStore database.MirrorSourceStore
+ userStore database.UserStore
+}
+
+type MirrorSourceComponent interface {
+ Create(ctx context.Context, req types.CreateMirrorSourceReq) (*database.MirrorSource, error)
+ Get(ctx context.Context, id int64, currentUser string) (*database.MirrorSource, error)
+ Index(ctx context.Context, currentUser string) ([]database.MirrorSource, error)
+ Update(ctx context.Context, req types.UpdateMirrorSourceReq) (*database.MirrorSource, error)
+ Delete(ctx context.Context, id int64, currentUser string) error
+}
+
+func NewMirrorSourceComponent(config *config.Config) (MirrorSourceComponent, error) {
+ return &mirrorSourceComponentImpl{
+ mirrorSourceStore: database.NewMirrorSourceStore(),
+ userStore: database.NewUserStore(),
+ }, nil
+}
+
+func (c *mirrorSourceComponentImpl) Create(ctx context.Context, req types.CreateMirrorSourceReq) (*database.MirrorSource, error) {
+ var ms database.MirrorSource
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, errors.New("user does not have admin permission")
+ }
+ ms.SourceName = req.SourceName
+ ms.InfoAPIUrl = req.InfoAPiUrl
+ res, err := c.mirrorSourceStore.Create(ctx, &ms)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create mirror source, error: %w", err)
+ }
+ return res, nil
+}
+
+func (c *mirrorSourceComponentImpl) Get(ctx context.Context, id int64, currentUser string) (*database.MirrorSource, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, errors.New("user does not have admin permission")
+ }
+ ms, err := c.mirrorSourceStore.Get(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get mirror source, error: %w", err)
+ }
+ return ms, nil
+}
+
+func (c *mirrorSourceComponentImpl) Index(ctx context.Context, currentUser string) ([]database.MirrorSource, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, errors.New("user does not have admin permission")
+ }
+ ms, err := c.mirrorSourceStore.Index(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get mirror source, error: %w", err)
+ }
+ return ms, nil
+}
+func (c *mirrorSourceComponentImpl) Update(ctx context.Context, req types.UpdateMirrorSourceReq) (*database.MirrorSource, error) {
+ var ms database.MirrorSource
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return nil, errors.New("user does not have admin permission")
+ }
+ ms.ID = req.ID
+ ms.SourceName = req.SourceName
+ ms.InfoAPIUrl = req.InfoAPiUrl
+ err = c.mirrorSourceStore.Update(ctx, &ms)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update mirror source, error: %w", err)
+ }
+ return &ms, nil
+}
+
+func (c *mirrorSourceComponentImpl) Delete(ctx context.Context, id int64, currentUser string) error {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ return errors.New("user does not have admin permission")
+ }
+ ms, err := c.mirrorSourceStore.Get(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to find mirror source, error: %w", err)
+ }
+ err = c.mirrorSourceStore.Delete(ctx, ms)
+ if err != nil {
+ return fmt.Errorf("failed to delete mirror source, error: %w", err)
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+const modelGitattributesContent = `*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.pdparams filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+`
+const LFSPrefix = "version https://git-lfs.github.com/spec/v1"
+
+type ModelComponent interface {
+ Index(ctx context.Context, filter *types.RepoFilter, per, page int, needOpWeight bool) ([]*types.Model, int, error)
+ Create(ctx context.Context, req *types.CreateModelReq) (*types.Model, error)
+ Update(ctx context.Context, req *types.UpdateModelReq) (*types.Model, error)
+ Delete(ctx context.Context, namespace, name, currentUser string) error
+ Show(ctx context.Context, namespace, name, currentUser string, needOpWeight bool) (*types.Model, error)
+ GetServerless(ctx context.Context, namespace, name, currentUser string) (*types.DeployRepo, error)
+ SDKModelInfo(ctx context.Context, namespace, name, ref, currentUser string) (*types.SDKModelInfo, error)
+ Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error)
+ SetRelationDatasets(ctx context.Context, req types.RelationDatasets) error
+ AddRelationDataset(ctx context.Context, req types.RelationDataset) error
+ DelRelationDataset(ctx context.Context, req types.RelationDataset) error
+ // create model deploy as inference/serverless
+ Deploy(ctx context.Context, deployReq types.DeployActReq, req types.ModelRunReq) (int64, error)
+ ListModelsByRuntimeFrameworkID(ctx context.Context, currentUser string, per, page int, id int64, deployType int) ([]types.Model, int, error)
+ ListAllByRuntimeFramework(ctx context.Context, currentUser string) ([]database.RuntimeFramework, error)
+ SetRuntimeFrameworkModes(ctx context.Context, deployType int, id int64, paths []string) ([]string, error)
+ DeleteRuntimeFrameworkModes(ctx context.Context, deployType int, id int64, paths []string) ([]string, error)
+ ListModelsOfRuntimeFrameworks(ctx context.Context, currentUser, search, sort string, per, page int, deployType int) ([]types.Model, int, error)
+ OrgModels(ctx context.Context, req *types.OrgModelsReq) ([]types.Model, int, error)
+}
+
+func NewModelComponent(config *config.Config) (ModelComponent, error) {
+ c := &modelComponentImpl{config: config}
+ var err error
+ c.repoComponent, err = NewRepoComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.spaceComponent, _ = NewSpaceComponent(config)
+ c.modelStore = database.NewModelStore()
+ c.repoStore = database.NewRepoStore()
+ c.spaceResourceStore = database.NewSpaceResourceStore()
+ c.userStore = database.NewUserStore()
+ c.userLikesStore = database.NewUserLikesStore()
+ c.deployer = deploy.NewDeployer()
+ c.tagStore = database.NewTagStore()
+ c.runtimeArchComponent, err = NewRuntimeArchitectureComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.accountingComponent, err = NewAccountingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.datasetStore = database.NewDatasetStore()
+ c.repoRuntimeFrameworkStore = database.NewRepositoriesRuntimeFramework()
+ c.runtimeFrameworksStore = database.NewRuntimeFrameworksStore()
+ c.deployTaskStore = database.NewDeployTaskStore()
+ c.gitServer, err = git.NewGitServer(config)
+ if err != nil {
+ return nil, err
+ }
+ c.userSvcClient = rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+ c.recomStore = database.NewRecomStore()
+ return c, nil
+}
+
+type modelComponentImpl struct {
+ config *config.Config
+ repoComponent RepoComponent
+ spaceComponent SpaceComponent
+ modelStore database.ModelStore
+ repoStore database.RepoStore
+ spaceResourceStore database.SpaceResourceStore
+ userStore database.UserStore
+ deployer deploy.Deployer
+ accountingComponent AccountingComponent
+ tagStore database.TagStore
+ runtimeArchComponent RuntimeArchitectureComponent
+ datasetStore database.DatasetStore
+ recomStore database.RecomStore
+ gitServer gitserver.GitServer
+ userLikesStore database.UserLikesStore
+ repoRuntimeFrameworkStore database.RepositoriesRuntimeFrameworkStore
+ deployTaskStore database.DeployTaskStore
+ runtimeFrameworksStore database.RuntimeFrameworksStore
+ userSvcClient rpc.UserSvcClient
+}
+
+func (c *modelComponentImpl) Index(ctx context.Context, filter *types.RepoFilter, per, page int, needOpWeight bool) ([]*types.Model, int, error) {
+ var (
+ err error
+ resModels []*types.Model
+ )
+ repos, total, err := c.repoComponent.PublicToUser(ctx, types.ModelRepo, filter.Username, filter, per, page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public model repos,error:%w", err)
+ return nil, 0, newError
+ }
+ var repoIDs []int64
+ for _, repo := range repos {
+ repoIDs = append(repoIDs, repo.ID)
+ }
+ models, err := c.modelStore.ByRepoIDs(ctx, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get models by repo ids,error:%w", err)
+ return nil, 0, newError
+ }
+
+ // loop through repos to keep the repos in sort order
+ for _, repo := range repos {
+ var model *database.Model
+ for _, m := range models {
+ if m.RepositoryID == repo.ID {
+ model = &m
+ break
+ }
+ }
+ if model == nil {
+ continue
+ }
+ var tags []types.RepoTag
+ for _, tag := range repo.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ resModels = append(resModels, &types.Model{
+ ID: model.ID,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ Likes: repo.Likes,
+ Downloads: repo.DownloadCount,
+ Path: repo.Path,
+ RepositoryID: repo.ID,
+ Private: repo.Private,
+ CreatedAt: model.CreatedAt,
+ Tags: tags,
+ UpdatedAt: repo.UpdatedAt,
+ Source: repo.Source,
+ SyncStatus: repo.SyncStatus,
+ License: repo.License,
+ Repository: common.BuildCloneInfo(c.config, model.Repository),
+ })
+ }
+ if needOpWeight {
+ opWeights, err := c.recomStore.LoadRepoOpWeights(ctx, repoIDs)
+ if err == nil {
+ //loop models and set RecomOpWeight
+ for _, model := range resModels {
+ if weight, ok := opWeights[model.RepositoryID]; ok {
+ model.RecomOpWeight = weight
+ }
+ }
+ }
+ }
+ return resModels, total, nil
+}
+
+func (c *modelComponentImpl) Create(ctx context.Context, req *types.CreateModelReq) (*types.Model, error) {
+ var (
+ nickname string
+ tags []types.RepoTag
+ )
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ if req.Nickname != "" {
+ nickname = req.Nickname
+ } else {
+ nickname = req.Name
+ }
+
+ if req.DefaultBranch == "" {
+ req.DefaultBranch = types.MainBranch
+ }
+ req.Nickname = nickname
+ req.RepoType = types.ModelRepo
+ req.Readme = generateReadmeData(req.License)
+ _, dbRepo, err := c.repoComponent.CreateRepo(ctx, req.CreateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ dbModel := database.Model{
+ Repository: dbRepo,
+ RepositoryID: dbRepo.ID,
+ BaseModel: req.BaseModel,
+ }
+
+ model, err := c.modelStore.Create(ctx, dbModel)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create database model, cause: %w", err)
+ }
+
+ // Create README.md file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: user.Username,
+ Email: user.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: req.Readme,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: readmeFileName,
+ }, types.ModelRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create README.md file, cause: %w", err)
+ }
+
+ // Create .gitattributes file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: user.Username,
+ Email: user.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: modelGitattributesContent,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: gitattributesFileName,
+ }, types.ModelRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create .gitattributes file, cause: %w", err)
+ }
+
+ for _, tag := range model.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ resModel := &types.Model{
+ ID: model.ID,
+ Name: model.Repository.Name,
+ Nickname: model.Repository.Nickname,
+ Description: model.Repository.Description,
+ Likes: model.Repository.Likes,
+ Downloads: model.Repository.DownloadCount,
+ Path: model.Repository.Path,
+ RepositoryID: model.RepositoryID,
+ Repository: common.BuildCloneInfo(c.config, model.Repository),
+ Private: model.Repository.Private,
+ User: &types.User{
+ Username: user.Username,
+ Nickname: user.NickName,
+ Email: user.Email,
+ },
+ Tags: tags,
+ CreatedAt: model.CreatedAt,
+ UpdatedAt: model.UpdatedAt,
+ BaseModel: model.BaseModel,
+ License: model.Repository.License,
+ }
+
+ return resModel, nil
+}
+
+func buildCreateFileReq(p *types.CreateFileParams, repoType types.RepositoryType) *types.CreateFileReq {
+ return &types.CreateFileReq{
+ Username: p.Username,
+ Email: p.Email,
+ Message: p.Message,
+ Branch: p.Branch,
+ Content: base64.StdEncoding.EncodeToString([]byte(p.Content)),
+ NewBranch: p.Branch,
+ Namespace: p.Namespace,
+ Name: p.Name,
+ FilePath: p.FilePath,
+ RepoType: repoType,
+ }
+}
+
+func (c *modelComponentImpl) Update(ctx context.Context, req *types.UpdateModelReq) (*types.Model, error) {
+ req.RepoType = types.ModelRepo
+ dbRepo, err := c.repoComponent.UpdateRepo(ctx, req.UpdateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ model, err := c.modelStore.ByRepoID(ctx, dbRepo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ if req.BaseModel != nil {
+ model.BaseModel = *req.BaseModel
+ }
+ model, err = c.modelStore.Update(ctx, *model)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update database model, error: %w", err)
+ }
+ resModel := &types.Model{
+ ID: model.ID,
+ Name: dbRepo.Name,
+ Nickname: dbRepo.Nickname,
+ Description: dbRepo.Description,
+ Likes: dbRepo.Likes,
+ Downloads: dbRepo.DownloadCount,
+ Path: dbRepo.Path,
+ RepositoryID: dbRepo.ID,
+ Private: dbRepo.Private,
+ CreatedAt: model.CreatedAt,
+ UpdatedAt: model.UpdatedAt,
+ BaseModel: model.BaseModel,
+ }
+
+ return resModel, nil
+}
+
+func (c *modelComponentImpl) Delete(ctx context.Context, namespace, name, currentUser string) error {
+ model, err := c.modelStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ deleteDatabaseRepoReq := types.DeleteRepoReq{
+ Username: currentUser,
+ Namespace: namespace,
+ Name: name,
+ RepoType: types.ModelRepo,
+ }
+ _, err = c.repoComponent.DeleteRepo(ctx, deleteDatabaseRepoReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete repo of model, error: %w", err)
+ }
+
+ err = c.modelStore.Delete(ctx, *model)
+ if err != nil {
+ return fmt.Errorf("failed to delete database model, error: %w", err)
+ }
+ return nil
+}
+
+func (c *modelComponentImpl) Show(ctx context.Context, namespace, name, currentUser string, needOpWeight bool) (*types.Model, error) {
+ var tags []types.RepoTag
+ model, err := c.modelStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, currentUser, model.Repository)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ ns, err := c.repoComponent.GetNameSpaceInfo(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace info for model, error: %w", err)
+ }
+
+ for _, tag := range model.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ likeExists, err := c.userLikesStore.IsExist(ctx, currentUser, model.Repository.ID)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user likes,error:%w", err)
+ return nil, newError
+ }
+
+ resModel := &types.Model{
+ ID: model.ID,
+ Name: model.Repository.Name,
+ Nickname: model.Repository.Nickname,
+ Description: model.Repository.Description,
+ Likes: model.Repository.Likes,
+ Downloads: model.Repository.DownloadCount,
+ Path: model.Repository.Path,
+ RepositoryID: model.Repository.ID,
+ DefaultBranch: model.Repository.DefaultBranch,
+ Repository: common.BuildCloneInfo(c.config, model.Repository),
+ Private: model.Repository.Private,
+ Tags: tags,
+ User: &types.User{
+ Username: model.Repository.User.Username,
+ Nickname: model.Repository.User.NickName,
+ Email: model.Repository.User.Email,
+ },
+ CreatedAt: model.CreatedAt,
+ UpdatedAt: model.Repository.UpdatedAt,
+ // TODO:default to ModelWidgetTypeGeneration, need to config later
+ WidgetType: types.ModelWidgetTypeGeneration,
+ UserLikes: likeExists,
+ Source: model.Repository.Source,
+ SyncStatus: model.Repository.SyncStatus,
+ BaseModel: model.BaseModel,
+ License: model.Repository.License,
+ MirrorLastUpdatedAt: model.Repository.Mirror.LastUpdatedAt,
+ CanWrite: permission.CanWrite,
+ CanManage: permission.CanAdmin,
+ Namespace: ns,
+ }
+ // admin user or owner can see the sensitive check status
+ if permission.CanAdmin {
+ resModel.SensitiveCheckStatus = model.Repository.SensitiveCheckStatus.String()
+ }
+ if needOpWeight {
+ opWeights, err := c.recomStore.LoadRepoOpWeights(ctx, []int64{model.RepositoryID})
+ if err == nil {
+ if weight, ok := opWeights[model.RepositoryID]; ok {
+ resModel.RecomOpWeight = weight
+ }
+ }
+ }
+ inferences, _ := c.repoRuntimeFrameworkStore.GetByRepoIDsAndType(ctx, model.Repository.ID, types.InferenceType)
+ if len(inferences) > 0 {
+ resModel.EnableInference = true
+ }
+ finetunes, _ := c.repoRuntimeFrameworkStore.GetByRepoIDsAndType(ctx, model.Repository.ID, types.FinetuneType)
+ if len(finetunes) > 0 {
+ resModel.EnableFinetune = true
+ }
+ evaluations, _ := c.repoRuntimeFrameworkStore.GetByRepoIDsAndType(ctx, model.Repository.ID, types.EvaluationType)
+ if len(evaluations) > 0 {
+ resModel.EnableEvaluation = true
+ }
+ return resModel, nil
+}
+
+func (c *modelComponentImpl) GetServerless(ctx context.Context, namespace, name, currentUser string) (*types.DeployRepo, error) {
+ model, err := c.modelStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model, error: %w", err)
+ }
+ allow, _ := c.repoComponent.AllowReadAccessRepo(ctx, model.Repository, currentUser)
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+ deploy, err := c.deployTaskStore.GetServerlessDeployByRepID(ctx, model.Repository.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get serverless deployment, error: %w", err)
+ }
+ if deploy == nil {
+ return nil, nil
+ }
+ endpoint, _ := c.repoComponent.GenerateEndpoint(ctx, deploy)
+
+ resDeploy := types.DeployRepo{
+ DeployID: deploy.ID,
+ DeployName: deploy.DeployName,
+ RepoID: deploy.RepoID,
+ SvcName: deploy.SvcName,
+ Status: deployStatusCodeToString(deploy.Status),
+ Hardware: deploy.Hardware,
+ Env: deploy.Env,
+ RuntimeFramework: deploy.RuntimeFramework,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ GitBranch: deploy.GitBranch,
+ ClusterID: deploy.ClusterID,
+ SecureLevel: deploy.SecureLevel,
+ CreatedAt: deploy.CreatedAt,
+ UpdatedAt: deploy.UpdatedAt,
+ ProxyEndpoint: endpoint,
+ }
+ return &resDeploy, nil
+}
+
+func (c *modelComponentImpl) SDKModelInfo(ctx context.Context, namespace, name, ref, currentUser string) (*types.SDKModelInfo, error) {
+ model, err := c.modelStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ allow, _ := c.repoComponent.AllowReadAccessRepo(ctx, model.Repository, currentUser)
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ var pipelineTag, libraryTag, sha string
+ var tags []string
+ for _, tag := range model.Repository.Tags {
+ tags = append(tags, tag.Name)
+ if tag.Category == "task" {
+ pipelineTag = tag.Name
+ }
+ if tag.Category == "framework" {
+ libraryTag = tag.Name
+ }
+ }
+
+ filePaths, err := getFilePaths(namespace, name, "", types.ModelRepo, ref, c.gitServer.GetRepoFileTree)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get all %s files, error: %w", types.ModelRepo, err)
+ }
+
+ var sdkFiles []types.SDKFile
+ for _, filePath := range filePaths {
+ sdkFiles = append(sdkFiles, types.SDKFile{Filename: filePath})
+ }
+
+ if ref == "" {
+ ref = model.Repository.DefaultBranch
+ }
+ getLastCommitReq := gitserver.GetRepoLastCommitReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: ref,
+ RepoType: types.ModelRepo,
+ }
+ lastCommit, err := c.gitServer.GetRepoLastCommit(ctx, getLastCommitReq)
+ if err != nil {
+ slog.Error("failed to get last commit", slog.String("namespace", namespace), slog.String("name", name), slog.String("ref", ref), slog.Any("error", err))
+ return nil, fmt.Errorf("failed to get last commit, error: %w", err)
+ }
+
+ relatedRepos, _ := c.repoComponent.RelatedRepos(ctx, model.RepositoryID, currentUser)
+ relatedSpaces := relatedRepos[types.SpaceRepo]
+ spaceNames := make([]string, len(relatedSpaces))
+ for idx, s := range relatedSpaces {
+ spaceNames[idx] = s.Name
+ }
+
+ if lastCommit != nil {
+ sha = lastCommit.ID
+ }
+
+ resModel := &types.SDKModelInfo{
+ ID: model.Repository.Path,
+ Author: model.Repository.User.Username,
+ Sha: sha,
+ CreatedAt: model.Repository.CreatedAt,
+ LastModified: model.Repository.UpdatedAt,
+ Private: model.Repository.Private,
+ Disabled: false,
+ Gated: nil,
+ Downloads: int(model.Repository.DownloadCount),
+ Likes: int(model.Repository.Likes),
+ LibraryName: libraryTag,
+ Tags: tags,
+ PipelineTag: pipelineTag,
+ MaskToken: "",
+ WidgetData: nil,
+ ModelIndex: nil,
+ Config: nil,
+ TransformersInfo: nil,
+ CardData: nil,
+ Siblings: sdkFiles,
+ Spaces: spaceNames,
+ SafeTensors: nil,
+ }
+
+ return resModel, nil
+}
+
+func (c *modelComponentImpl) Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error) {
+ model, err := c.modelStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ allow, _ := c.repoComponent.AllowReadAccessRepo(ctx, model.Repository, currentUser)
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ return c.getRelations(ctx, model.RepositoryID, currentUser)
+}
+
+func (c *modelComponentImpl) SetRelationDatasets(ctx context.Context, req types.RelationDatasets) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist, %w", err)
+ }
+
+ if !user.CanAdmin() {
+ return fmt.Errorf("only admin is allowed to set dataset for model")
+ }
+
+ _, err = c.repoStore.FindByPath(ctx, types.ModelRepo, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: REPOCARD_FILENAME,
+ RepoType: types.ModelRepo,
+ }
+
+ metaMap, splits, err := GetMetaMapFromReadMe(c.gitServer, getFileContentReq)
+ if err != nil {
+ return fmt.Errorf("failed parse meta from readme, cause: %w", err)
+ }
+ metaMap["datasets"] = req.Datasets
+ output, err := GetOutputForReadme(metaMap, splits)
+ if err != nil {
+ return fmt.Errorf("failed generate output for readme, cause: %w", err)
+ }
+
+ var readmeReq types.UpdateFileReq
+ readmeReq.Branch = types.MainBranch
+ readmeReq.Message = "update dataset tags"
+ readmeReq.FilePath = REPOCARD_FILENAME
+ readmeReq.RepoType = types.ModelRepo
+ readmeReq.Namespace = req.Namespace
+ readmeReq.Name = req.Name
+ readmeReq.Username = req.CurrentUser
+ readmeReq.Email = user.Email
+ readmeReq.Content = base64.StdEncoding.EncodeToString([]byte(output))
+
+ err = c.gitServer.UpdateRepoFile(&readmeReq)
+ if err != nil {
+ return fmt.Errorf("failed to set dataset tag to %s file, cause: %w", readmeReq.FilePath, err)
+ }
+
+ return nil
+}
+
+func (c *modelComponentImpl) AddRelationDataset(ctx context.Context, req types.RelationDataset) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist, %w", err)
+ }
+
+ if !user.CanAdmin() {
+ return fmt.Errorf("only admin was allowed to set dataset for model")
+ }
+
+ _, err = c.repoStore.FindByPath(ctx, types.ModelRepo, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: "main",
+ Path: REPOCARD_FILENAME,
+ RepoType: types.ModelRepo,
+ }
+ metaMap, splits, err := GetMetaMapFromReadMe(c.gitServer, getFileContentReq)
+ if err != nil {
+ return fmt.Errorf("failed parse meta from readme, cause: %w", err)
+ }
+ datasets, ok := metaMap["datasets"]
+ if !ok {
+ datasets = []string{req.Dataset}
+ } else {
+ datasets = append(datasets.([]interface{}), req.Dataset)
+ }
+ metaMap["datasets"] = datasets
+ output, err := GetOutputForReadme(metaMap, splits)
+ if err != nil {
+ return fmt.Errorf("failed generate output for readme, cause: %w", err)
+ }
+
+ var readmeReq types.UpdateFileReq
+ readmeReq.Branch = "main"
+ readmeReq.Message = "add relation dataset"
+ readmeReq.FilePath = REPOCARD_FILENAME
+ readmeReq.RepoType = types.ModelRepo
+ readmeReq.Namespace = req.Namespace
+ readmeReq.Name = req.Name
+ readmeReq.Username = req.CurrentUser
+ readmeReq.Email = user.Email
+ readmeReq.Content = base64.StdEncoding.EncodeToString([]byte(output))
+
+ err = c.gitServer.UpdateRepoFile(&readmeReq)
+ if err != nil {
+ return fmt.Errorf("failed to add dataset tag to %s file, cause: %w", readmeReq.FilePath, err)
+ }
+
+ return nil
+}
+
+func (c *modelComponentImpl) DelRelationDataset(ctx context.Context, req types.RelationDataset) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist, %w", err)
+ }
+
+ if !user.CanAdmin() {
+ return fmt.Errorf("only admin was allowed to delete dataset for model")
+ }
+
+ _, err = c.repoStore.FindByPath(ctx, types.ModelRepo, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find model, error: %w", err)
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: "main",
+ Path: REPOCARD_FILENAME,
+ RepoType: types.ModelRepo,
+ }
+ metaMap, splits, err := GetMetaMapFromReadMe(c.gitServer, getFileContentReq)
+ if err != nil {
+ return fmt.Errorf("failed parse meta from readme, cause: %w", err)
+ }
+ datasets, ok := metaMap["datasets"]
+ if !ok {
+ return nil
+ } else {
+ var newDatasets []string
+ for _, v := range datasets.([]interface{}) {
+ if v.(string) != req.Dataset {
+ newDatasets = append(newDatasets, v.(string))
+ }
+ }
+ metaMap["datasets"] = newDatasets
+ }
+ output, err := GetOutputForReadme(metaMap, splits)
+ if err != nil {
+ return fmt.Errorf("failed generate output for readme, cause: %w", err)
+ }
+
+ var readmeReq types.UpdateFileReq
+ readmeReq.Branch = "main"
+ readmeReq.Message = "delete relation dataset"
+ readmeReq.FilePath = REPOCARD_FILENAME
+ readmeReq.RepoType = types.ModelRepo
+ readmeReq.Namespace = req.Namespace
+ readmeReq.Name = req.Name
+ readmeReq.Username = req.CurrentUser
+ readmeReq.Email = user.Email
+ readmeReq.Content = base64.StdEncoding.EncodeToString([]byte(output))
+
+ err = c.gitServer.UpdateRepoFile(&readmeReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete dataset tag to %s file, cause: %w", readmeReq.FilePath, err)
+ }
+
+ return nil
+}
+
+func (c *modelComponentImpl) getRelations(ctx context.Context, fromRepoID int64, currentUser string) (*types.Relations, error) {
+ res, err := c.repoComponent.RelatedRepos(ctx, fromRepoID, currentUser)
+ if err != nil {
+ return nil, err
+ }
+ rels := new(types.Relations)
+ datasetRepos := res[types.DatasetRepo]
+ for _, repo := range datasetRepos {
+ rels.Datasets = append(rels.Datasets, &types.Dataset{
+ Path: repo.Path,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ UpdatedAt: repo.UpdatedAt,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ })
+ }
+ codeRepos := res[types.CodeRepo]
+ for _, repo := range codeRepos {
+ rels.Codes = append(rels.Codes, &types.Code{
+ Path: repo.Path,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ UpdatedAt: repo.UpdatedAt,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ })
+ }
+ spaceRepos := res[types.SpaceRepo]
+ spacePaths := make([]string, 0)
+ for _, repo := range spaceRepos {
+ spacePaths = append(spacePaths, repo.Path)
+ }
+ spaces, err := c.spaceComponent.ListByPath(ctx, spacePaths)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get space info by paths, error: %w", err)
+ }
+ rels.Spaces = spaces
+
+ promptRepos := res[types.PromptRepo]
+ for _, repo := range promptRepos {
+ rels.Prompts = append(rels.Prompts, &types.PromptRes{
+ Path: repo.Path,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ UpdatedAt: repo.UpdatedAt,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ })
+ }
+ return rels, nil
+}
+
+func GetFilePathObjects(namespace, repoName, folder string, repoType types.RepositoryType, ref string, gsTree func(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error)) ([]*types.File, error) {
+ allFiles, err := getAllFiles(namespace, repoName, folder, repoType, ref, gsTree)
+ if err != nil {
+ return nil, err
+ }
+ return allFiles, nil
+}
+
+func getFilePaths(namespace, repoName, folder string, repoType types.RepositoryType, ref string, gsTree func(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error)) ([]string, error) {
+ var filePaths []string
+ allFiles, err := getAllFiles(namespace, repoName, folder, repoType, ref, gsTree)
+ if err != nil {
+ return nil, err
+ }
+ for _, f := range allFiles {
+ filePaths = append(filePaths, f.Path)
+ }
+
+ return filePaths, nil
+}
+
+// create model deploy as inference/serverless
+func (c *modelComponentImpl) Deploy(ctx context.Context, deployReq types.DeployActReq, req types.ModelRunReq) (int64, error) {
+ m, err := c.modelStore.FindByPath(ctx, deployReq.Namespace, deployReq.Name)
+ if err != nil {
+ return -1, fmt.Errorf("cannot find model, %w", err)
+ }
+ if deployReq.DeployType == types.ServerlessType {
+ // only one service deploy was allowed
+ d, err := c.deployTaskStore.GetServerlessDeployByRepID(ctx, m.Repository.ID)
+ if err != nil {
+ return -1, fmt.Errorf("fail to get deploy, %w", err)
+ }
+ if d != nil {
+ return d.ID, nil
+ }
+ }
+ // found user id
+ user, err := c.userStore.FindByUsername(ctx, deployReq.CurrentUser)
+ if err != nil {
+ return -1, fmt.Errorf("cannot find user for deploy model, %w", err)
+ }
+
+ if deployReq.DeployType == types.ServerlessType {
+ // Check if the user is an admin
+ isAdmin := c.repoComponent.IsAdminRole(user)
+ if !isAdmin {
+ return -1, fmt.Errorf("need admin permission for Serverless deploy")
+ }
+ }
+
+ frame, err := c.runtimeFrameworksStore.FindEnabledByID(ctx, req.RuntimeFrameworkID)
+ if err != nil {
+ return -1, fmt.Errorf("cannot find available runtime framework, %w", err)
+ }
+
+ // put repo-type and namespace/name in annotation
+ annotations := make(map[string]string)
+ annotations[types.ResTypeKey] = string(types.ModelRepo)
+ annotations[types.ResNameKey] = fmt.Sprintf("%s/%s", deployReq.Namespace, deployReq.Name)
+ annoStr, err := json.Marshal(annotations)
+ if err != nil {
+ return -1, fmt.Errorf("fail to create annotations for deploy model, %w", err)
+ }
+
+ resource, err := c.spaceResourceStore.FindByID(ctx, req.ResourceID)
+ if err != nil {
+ return -1, fmt.Errorf("cannot find resource, %w", err)
+ }
+
+ // check if there is price for the resource
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)),
+ ResourceID: strconv.FormatInt(int64(resource.ID), 10),
+ })
+ if err != nil {
+ return -1, fmt.Errorf("fail to find price data, %w", err)
+ }
+ if priceData.Total == 0 {
+ return -1, fmt.Errorf("cannot find valid price data")
+ }
+ // check user balance
+ if priceData.Prices[0].SkuPrice > 0 && req.OrderDetailID == 0 {
+ account, err := c.accountingComponent.QueryBalanceByUserIDInternal(ctx, deployReq.CurrentUser)
+ if err != nil {
+ return -1, fmt.Errorf("cannot find user balance, %w", err)
+ }
+ if account.Balance <= 0 {
+ return -1, fmt.Errorf("balance is not enough to run resources. current balance: %.2f", account.Balance/100)
+ }
+ }
+
+ var hardware types.HardWare
+ err = json.Unmarshal([]byte(resource.Resources), &hardware)
+ if err != nil {
+ return -1, fmt.Errorf("invalid hardware setting, %w", err)
+ }
+
+ _, err = c.deployer.CheckResourceAvailable(ctx, req.ClusterID, req.OrderDetailID, &hardware)
+ if err != nil {
+ return -1, fmt.Errorf("fail to check resource, %w", err)
+ }
+
+ // choose image
+ containerImg := frame.FrameCpuImage
+ if hardware.Npu.Num != "" {
+ // use npu image
+ containerImg = frame.FrameNpuImage
+ } else if hardware.Gpu.Num != "" {
+ // use gpu image
+ containerImg = frame.FrameImage
+ }
+
+ // create deploy for model
+ return c.deployer.Deploy(ctx, types.DeployRepo{
+ DeployName: req.DeployName,
+ SpaceID: 0,
+ Path: m.Repository.Path,
+ GitPath: m.Repository.GitPath,
+ GitBranch: req.Revision,
+ Env: req.Env,
+ Hardware: resource.Resources,
+ UserID: user.ID,
+ ModelID: m.ID,
+ RepoID: m.Repository.ID,
+ RuntimeFramework: frame.FrameName,
+ ContainerPort: frame.ContainerPort, // default container port
+ ImageID: containerImg, // do not need build pod image for model
+ MinReplica: req.MinReplica,
+ MaxReplica: req.MaxReplica,
+ Annotation: string(annoStr),
+ ClusterID: req.ClusterID,
+ SecureLevel: req.SecureLevel,
+ Type: deployReq.DeployType,
+ UserUUID: user.UUID,
+ SKU: strconv.FormatInt(resource.ID, 10),
+ OrderDetailID: req.OrderDetailID,
+ })
+}
+
+func (c *modelComponentImpl) ListModelsByRuntimeFrameworkID(ctx context.Context, currentUser string, per, page int, id int64, deployType int) ([]types.Model, int, error) {
+ var (
+ user database.User
+ err error
+ resModels []types.Model
+ )
+ if currentUser != "" {
+ user, err = c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get current user,error:%w", err)
+ }
+ }
+
+ runtimeRepos, err := c.repoRuntimeFrameworkStore.ListByRuntimeFrameworkID(ctx, id, deployType)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get repo by runtime,error:%w", err)
+ }
+
+ if runtimeRepos == nil {
+ return nil, 0, nil
+ }
+
+ var repoIDs []int64
+ for _, repo := range runtimeRepos {
+ repoIDs = append(repoIDs, repo.RepoID)
+ }
+
+ repos, total, err := c.repoStore.ListRepoPublicToUserByRepoIDs(ctx, types.ModelRepo, user.ID, "", "", per, page, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public model repos,error:%w", err)
+ return nil, 0, newError
+ }
+
+ for _, repo := range repos {
+ resModels = append(resModels, types.Model{
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ Path: repo.Path,
+ RepositoryID: repo.ID,
+ Private: repo.Private,
+ })
+ }
+ return resModels, total, nil
+}
+
+func (c *modelComponentImpl) ListAllByRuntimeFramework(ctx context.Context, currentUser string) ([]database.RuntimeFramework, error) {
+ runtimes, err := c.runtimeFrameworksStore.ListAll(ctx)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public model repos,error:%w", err)
+ return nil, newError
+ }
+
+ return runtimes, nil
+}
+
+func (c *modelComponentImpl) SetRuntimeFrameworkModes(ctx context.Context, deployType int, id int64, paths []string) ([]string, error) {
+ runtimeRepos, err := c.runtimeFrameworksStore.FindByID(ctx, id)
+ if err != nil {
+ return nil, err
+ }
+
+ if runtimeRepos == nil {
+ return nil, fmt.Errorf("failed to get runtime framework")
+ }
+
+ models, err := c.modelStore.ListByPath(ctx, paths)
+ if err != nil {
+ return nil, err
+ }
+
+ runtime_framework_tags, _ := c.tagStore.GetTagsByScopeAndCategories(ctx, "model", []string{"runtime_framework", "resource"})
+
+ var failedModels []string
+ for _, model := range models {
+ relations, err := c.repoRuntimeFrameworkStore.GetByIDsAndType(ctx, id, model.Repository.ID, deployType)
+ if err != nil {
+ return nil, err
+ }
+ if relations == nil || len(relations) < 1 {
+ err = c.repoRuntimeFrameworkStore.Add(ctx, id, model.Repository.ID, deployType)
+ if err != nil {
+ failedModels = append(failedModels, model.Repository.Path)
+ }
+ _, modelName := model.Repository.NamespaceAndName()
+ err = c.runtimeArchComponent.AddRuntimeFrameworkTag(ctx, runtime_framework_tags, model.Repository.ID, id)
+ if err != nil {
+ return nil, err
+ }
+ err = c.runtimeArchComponent.AddResourceTag(ctx, runtime_framework_tags, modelName, model.Repository.ID)
+ if err != nil {
+ return nil, err
+ }
+ }
+ }
+
+ return failedModels, nil
+}
+
+func (c *modelComponentImpl) DeleteRuntimeFrameworkModes(ctx context.Context, deployType int, id int64, paths []string) ([]string, error) {
+ models, err := c.modelStore.ListByPath(ctx, paths)
+ if err != nil {
+ return nil, err
+ }
+
+ var failedModels []string
+ for _, model := range models {
+ err = c.repoRuntimeFrameworkStore.Delete(ctx, id, model.Repository.ID, deployType)
+ if err != nil {
+ failedModels = append(failedModels, model.Repository.Path)
+ }
+ }
+
+ return failedModels, nil
+}
+
+func (c *modelComponentImpl) ListModelsOfRuntimeFrameworks(ctx context.Context, currentUser, search, sort string, per, page int, deployType int) ([]types.Model, int, error) {
+ var (
+ user database.User
+ err error
+ resModels []types.Model
+ )
+
+ user, err = c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get current user %s, error:%w", currentUser, err)
+ }
+
+ runtimeRepos, err := c.repoRuntimeFrameworkStore.ListRepoIDsByType(ctx, deployType)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get repo by deploy type, error:%w", err)
+ }
+
+ if runtimeRepos == nil || len(runtimeRepos) < 1 {
+ return nil, 0, nil
+ }
+
+ var repoIDs []int64
+ for _, repo := range runtimeRepos {
+ repoIDs = append(repoIDs, repo.RepoID)
+ }
+
+ repos, total, err := c.repoStore.ListRepoPublicToUserByRepoIDs(ctx, types.ModelRepo, user.ID, search, sort, per, page, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public model repos, error:%w", err)
+ return nil, 0, newError
+ }
+ // define EnableInference
+ enableInference := deployType == types.InferenceType
+ enableFinetune := deployType == types.FinetuneType
+ enableEvaluation := deployType == types.EvaluationType
+
+ for _, repo := range repos {
+ resModels = append(resModels, types.Model{
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ Path: repo.Path,
+ RepositoryID: repo.ID,
+ Private: repo.Private,
+ EnableInference: enableInference,
+ EnableFinetune: enableFinetune,
+ EnableEvaluation: enableEvaluation,
+ })
+ }
+ return resModels, total, nil
+}
+
+func (c *modelComponentImpl) OrgModels(ctx context.Context, req *types.OrgModelsReq) ([]types.Model, int, error) {
+ var resModels []types.Model
+ var err error
+ r := membership.RoleUnknown
+ if req.CurrentUser != "" {
+ r, err = c.userSvcClient.GetMemberRole(ctx, req.Namespace, req.CurrentUser)
+ // log error, and treat user as unknown role in org
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Namespace), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ }
+ onlyPublic := !r.CanRead()
+ ms, total, err := c.modelStore.ByOrgPath(ctx, req.Namespace, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user datasets,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range ms {
+ resModels = append(resModels, types.Model{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resModels, total, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log/slog"
+ "path"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/multisync"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+type multiSyncComponentImpl struct {
+ multiSyncStore database.MultiSyncStore
+ repoStore database.RepoStore
+ modelStore database.ModelStore
+ datasetStore database.DatasetStore
+ namespaceStore database.NamespaceStore
+ userStore database.UserStore
+ syncVersionStore database.SyncVersionStore
+ tagStore database.TagStore
+ fileStore database.FileStore
+ gitServer gitserver.GitServer
+}
+
+type MultiSyncComponent interface {
+ More(ctx context.Context, cur int64, limit int64) ([]types.SyncVersion, error)
+ SyncAsClient(ctx context.Context, sc multisync.Client) error
+}
+
+func NewMultiSyncComponent(config *config.Config) (MultiSyncComponent, error) {
+ git, err := git.NewGitServer(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server: %w", err)
+ }
+ return &multiSyncComponentImpl{
+ multiSyncStore: database.NewMultiSyncStore(),
+ repoStore: database.NewRepoStore(),
+ modelStore: database.NewModelStore(),
+ datasetStore: database.NewDatasetStore(),
+ namespaceStore: database.NewNamespaceStore(),
+ userStore: database.NewUserStore(),
+ syncVersionStore: database.NewSyncVersionStore(),
+ tagStore: database.NewTagStore(),
+ fileStore: database.NewFileStore(),
+ gitServer: git,
+ }, nil
+}
+
+func (c *multiSyncComponentImpl) More(ctx context.Context, cur int64, limit int64) ([]types.SyncVersion, error) {
+ dbVersions, err := c.multiSyncStore.GetAfter(ctx, cur, limit)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get sync versions after %d from db: %w", cur, err)
+ }
+ var versions []types.SyncVersion
+ for _, v := range dbVersions {
+ versions = append(versions, types.SyncVersion{
+ Version: v.Version,
+ SourceID: v.SourceID,
+ RepoPath: v.RepoPath,
+ RepoType: v.RepoType,
+ LastModifyTime: v.LastModifiedAt,
+ ChangeLog: v.ChangeLog,
+ })
+ }
+ return versions, nil
+}
+
+func (c *multiSyncComponentImpl) SyncAsClient(ctx context.Context, sc multisync.Client) error {
+ var currentVersion int64
+ v, err := c.multiSyncStore.GetLatest(ctx)
+ if err != nil {
+ if err != sql.ErrNoRows {
+ return fmt.Errorf("failed to get latest sync version from db: %w", err)
+ }
+ }
+
+ currentVersion = v.Version
+ var hasMore = true
+ for hasMore {
+ ctxWithTimeout, cancel := context.WithTimeout(ctx, 60*time.Second)
+ resp, err := sc.Latest(ctxWithTimeout, currentVersion)
+ cancel()
+ if err != nil {
+ return fmt.Errorf("failed to sync latest version from client, current version:%d, error: %w", currentVersion, err)
+ }
+ //create local repo
+ for _, v := range resp.Data.Versions {
+ err := c.createLocalSyncVersion(ctx, v)
+ if err != nil {
+ slog.Error("failed to create database sync version", slog.Any("sync_version", v), slog.Any("error", err))
+ continue
+ }
+ }
+
+ hasMore = resp.Data.HasMore
+ if len(resp.Data.Versions) > 0 {
+ currentVersion = resp.Data.Versions[len(resp.Data.Versions)-1].Version
+ }
+ }
+
+ syncVersions, err := c.multiSyncStore.GetAfterDistinct(ctx, v.Version)
+ if err != nil {
+ slog.Error("failed to find distinct sync versions", slog.Any("error", err))
+ return err
+ }
+ for _, v := range syncVersions {
+ sv := types.SyncVersion{
+ Version: v.Version,
+ SourceID: v.SourceID,
+ RepoPath: v.RepoPath,
+ RepoType: v.RepoType,
+ LastModifyTime: v.LastModifiedAt,
+ ChangeLog: v.ChangeLog,
+ }
+ switch v.RepoType {
+ case types.ModelRepo:
+ ctxGetModel, cancel := context.WithTimeout(ctx, 10*time.Second)
+ modelInfo, err := sc.ModelInfo(ctxGetModel, sv)
+ if err != nil {
+ slog.Error("failed to get model info from client", slog.Any("sync_version", v))
+ continue
+ }
+ ReadMeData, err := sc.ReadMeData(ctxGetModel, sv)
+ if err != nil {
+ slog.Error("failed to get model readme from client", slog.Any("sync_version", v), slog.Any("error", err))
+ }
+ cancel()
+ modelInfo.Readme = ReadMeData
+ ctxCreateModel, cancel := context.WithTimeout(ctx, 5*time.Second)
+ err = c.createLocalModel(ctxCreateModel, modelInfo, sv, sc)
+ cancel()
+ if err != nil {
+ slog.Error("failed to create local synced repo", slog.Any("sync_version", v), slog.Any("error", err))
+ }
+ case types.DatasetRepo:
+ ctxGetDataset, cancel := context.WithTimeout(ctx, 10*time.Second)
+ datasetInfo, err := sc.DatasetInfo(ctxGetDataset, sv)
+ if err != nil {
+ slog.Error("failed to get model info from client", slog.Any("sync_version", v))
+ continue
+ }
+ ReadMeData, err := sc.ReadMeData(ctxGetDataset, sv)
+ if err != nil {
+ slog.Error("failed to get model readme from client", slog.Any("sync_version", v), slog.Any("error", err))
+ }
+ cancel()
+ datasetInfo.Readme = ReadMeData
+ ctxCreateDataset, cancel := context.WithTimeout(ctx, 5*time.Second)
+ err = c.createLocalDataset(ctxCreateDataset, datasetInfo, sv, sc)
+ cancel()
+ if err != nil {
+ slog.Error("failed to create local synced repo", slog.Any("sync_version", v), slog.Any("error", err))
+ }
+ default:
+ slog.Error("failed to create local synced repo, unsupported repo type", slog.Any("sync_version", v), slog.Any("error", err))
+ }
+ }
+
+ return nil
+}
+
+func (c *multiSyncComponentImpl) createLocalDataset(ctx context.Context, m *types.Dataset, s types.SyncVersion, sc multisync.Client) error {
+ namespace, name, _ := strings.Cut(m.Path, "/")
+ //add prefix to avoid namespace conflict
+ namespace = common.AddPrefixBySourceID(s.SourceID, namespace)
+
+ //use namespace as the user login name
+ userName := namespace
+ var user database.User
+ user, err := c.getUser(ctx, userName)
+ if err != nil {
+ if err != sql.ErrNoRows {
+ return fmt.Errorf("fail to get user, userName:%s, error: %w", userName, err)
+ }
+ }
+ //user not exists, create new one
+ if user.ID == 0 {
+ //create as user instead of org, no matter if the namespace is org or user
+ user, err = c.createUser(ctx, types.CreateUserRequest{
+ Name: m.User.Nickname,
+ Username: userName,
+ Email: common.AddPrefixBySourceID(s.SourceID, m.User.Email),
+ })
+ if err != nil {
+ return fmt.Errorf("fail to create user for namespace, namespace:%s, error: %w", namespace, err)
+ }
+ }
+ //create new database repo
+ dbRepo := database.Repository{
+ UserID: user.ID,
+ //new path with prefixed namespace
+ Path: path.Join(namespace, name),
+ GitPath: fmt.Sprintf("%ss_%s/%s", types.DatasetRepo, namespace, name),
+ Name: name,
+ Nickname: m.Nickname,
+ Description: m.Description,
+ Private: m.Private,
+ Readme: m.Readme,
+ // License: req.License,
+ DefaultBranch: m.DefaultBranch,
+ RepositoryType: types.DatasetRepo,
+ Source: types.OpenCSGSource,
+ SyncStatus: types.SyncStatusPending,
+ // HTTPCloneURL: gitRepo.HttpCloneURL,
+ // SSHCloneURL: gitRepo.SshCloneURL,
+ }
+ newDBRepo, err := c.repoStore.UpdateOrCreateRepo(ctx, dbRepo)
+ if err != nil {
+ return fmt.Errorf("fail to create database repo, error: %w", err)
+ }
+
+ if len(m.Tags) > 0 {
+ var repoTags []database.RepositoryTag
+ for _, tag := range m.Tags {
+ dbTag := database.Tag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ Scope: database.DatasetTagScope,
+ }
+ t, err := c.tagStore.FindOrCreate(ctx, dbTag)
+ if err != nil {
+ slog.Error("failed to create or find database tag", slog.Any("tag", dbTag))
+ continue
+ }
+ repoTags = append(repoTags, database.RepositoryTag{
+ RepositoryID: newDBRepo.ID,
+ TagID: t.ID,
+ })
+ }
+
+ err = c.repoStore.DeleteAllTags(ctx, newDBRepo.ID)
+ if err != nil && err != sql.ErrNoRows {
+ slog.Error("failed to delete database tag", slog.Any("error", err))
+ }
+
+ err = c.repoStore.BatchCreateRepoTags(ctx, repoTags)
+ if err != nil {
+ slog.Error("failed to create database tag", slog.Any("error", err))
+ }
+ }
+
+ err = c.repoStore.DeleteAllFiles(ctx, newDBRepo.ID)
+ if err != nil && err != sql.ErrNoRows {
+ slog.Error("failed to delete database files", slog.Any("error", err))
+ }
+
+ ctxGetFileList, cancel := context.WithTimeout(ctx, 5*time.Second)
+ files, err := sc.FileList(ctxGetFileList, s)
+ cancel()
+ if err != nil && err != sql.ErrNoRows {
+ slog.Error("failed to get all files of repo", slog.Any("sync_version", s), slog.Any("error", err))
+ }
+ if len(files) > 0 {
+ var dbFiles []database.File
+ for _, f := range files {
+ dbFiles = append(dbFiles, database.File{
+ Name: f.Name,
+ Path: f.Path,
+ ParentPath: common.ConvertDotToSlash(filepath.Dir(f.Path)),
+ Size: f.Size,
+ LastCommitMessage: f.Commit.Message,
+ LastCommitDate: f.Commit.CommitterDate,
+ RepositoryID: newDBRepo.ID,
+ })
+ }
+
+ err = c.fileStore.BatchCreate(ctx, dbFiles)
+ if err != nil {
+ slog.Error("failed to create all files of repo", slog.Any("sync_version", s))
+ }
+ }
+
+ //create new dataset record related to repo
+ dbDataset := database.Dataset{
+ Repository: newDBRepo,
+ RepositoryID: newDBRepo.ID,
+ }
+ _, err = c.datasetStore.CreateIfNotExist(ctx, dbDataset)
+ if err != nil {
+ return fmt.Errorf("failed to create dataset in db, cause: %w", err)
+ }
+ return nil
+
+}
+func (c *multiSyncComponentImpl) createLocalModel(ctx context.Context, m *types.Model, s types.SyncVersion, sc multisync.Client) error {
+ namespace, name, _ := strings.Cut(m.Path, "/")
+ //add prefix to avoid namespace conflict
+ namespace = common.AddPrefixBySourceID(s.SourceID, namespace)
+
+ //use namespace as the user login name
+ userName := namespace
+ var user database.User
+ user, err := c.getUser(ctx, userName)
+ if err != nil {
+ if err != sql.ErrNoRows {
+ return fmt.Errorf("fail to get user, userName:%s, error: %w", userName, err)
+ }
+ }
+ //user not exists, create new one
+ if user.ID == 0 {
+ //create as user instead of org, no matter if the namespace is org or user
+ user, err = c.createUser(ctx, types.CreateUserRequest{
+ Name: m.User.Nickname,
+ Username: userName,
+ Email: common.AddPrefixBySourceID(s.SourceID, m.User.Email),
+ })
+ if err != nil {
+ return fmt.Errorf("fail to create user for namespace, namespace:%s, error: %w", namespace, err)
+ }
+ }
+ //create new database repo
+ dbRepo := database.Repository{
+ UserID: user.ID,
+ //new path with prefixed namespace
+ Path: path.Join(namespace, name),
+ GitPath: fmt.Sprintf("%ss_%s/%s", types.ModelRepo, namespace, name),
+ Name: name,
+ Nickname: m.Nickname,
+ Description: m.Description,
+ Private: m.Private,
+ Readme: m.Readme,
+ // License: req.License,
+ DefaultBranch: m.DefaultBranch,
+ RepositoryType: types.ModelRepo,
+ Source: types.OpenCSGSource,
+ SyncStatus: types.SyncStatusPending,
+ // HTTPCloneURL: gitRepo.HttpCloneURL,
+ // SSHCloneURL: gitRepo.SshCloneURL,
+ }
+ newDBRepo, err := c.repoStore.UpdateOrCreateRepo(ctx, dbRepo)
+ if err != nil {
+ return fmt.Errorf("fail to create database repo, error: %w", err)
+ }
+
+ if len(m.Tags) > 0 {
+ var repoTags []database.RepositoryTag
+ for _, tag := range m.Tags {
+ dbTag := database.Tag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ Scope: database.ModelTagScope,
+ }
+ t, err := c.tagStore.FindOrCreate(ctx, dbTag)
+ if err != nil {
+ slog.Error("failed to create or find database tag", slog.Any("tag", dbTag))
+ continue
+ }
+ repoTags = append(repoTags, database.RepositoryTag{
+ RepositoryID: newDBRepo.ID,
+ TagID: t.ID,
+ })
+ }
+ err = c.repoStore.DeleteAllTags(ctx, newDBRepo.ID)
+ if err != nil && err != sql.ErrNoRows {
+ slog.Error("failed to delete database tag", slog.Any("error", err))
+ }
+ err = c.repoStore.BatchCreateRepoTags(ctx, repoTags)
+ if err != nil {
+ slog.Error("failed to batch create database tag", slog.Any("error", err))
+ }
+ }
+
+ err = c.repoStore.DeleteAllFiles(ctx, newDBRepo.ID)
+ if err != nil && err != sql.ErrNoRows {
+ slog.Error("failed to delete all files for repo", slog.Any("error", err))
+ }
+
+ ctxGetFileList, cancel := context.WithTimeout(ctx, 5*time.Second)
+ files, err := sc.FileList(ctxGetFileList, s)
+ cancel()
+ if err != nil && err != sql.ErrNoRows {
+ slog.Error("failed to get all files of repo", slog.Any("sync_version", s), slog.Any("error", err))
+ }
+ if len(files) > 0 {
+ var dbFiles []database.File
+ for _, f := range files {
+ dbFiles = append(dbFiles, database.File{
+ Name: f.Name,
+ Path: f.Path,
+ ParentPath: common.ConvertDotToSlash(filepath.Dir(f.Path)),
+ Size: f.Size,
+ LastCommitMessage: f.Commit.Message,
+ LastCommitDate: f.Commit.CommitterDate,
+ RepositoryID: newDBRepo.ID,
+ })
+ }
+
+ err = c.fileStore.BatchCreate(ctx, dbFiles)
+ if err != nil {
+ slog.Error("failed to create all files of repo", slog.Any("sync_version", s))
+ }
+ }
+
+ //create new model record related to repo
+ dbModel := database.Model{
+ Repository: newDBRepo,
+ RepositoryID: newDBRepo.ID,
+ BaseModel: m.BaseModel,
+ }
+ _, err = c.modelStore.CreateIfNotExist(ctx, dbModel)
+ if err != nil {
+ return fmt.Errorf("failed to create database model, cause: %w", err)
+ }
+ return nil
+}
+
+func (c *multiSyncComponentImpl) createUser(ctx context.Context, req types.CreateUserRequest) (database.User, error) {
+ gsUserReq := gitserver.CreateUserRequest{
+ Nickname: req.Name,
+ Username: req.Username,
+ Email: req.Email,
+ }
+ gsUserResp, err := c.gitServer.CreateUser(gsUserReq)
+ if err != nil {
+ newError := fmt.Errorf("failed to create gitserver user,error:%w", err)
+ return database.User{}, newError
+ }
+
+ namespace := &database.Namespace{
+ Path: req.Username,
+ Mirrored: true,
+ }
+ user := &database.User{
+ NickName: req.Name,
+ Username: req.Username,
+ Email: req.Email,
+ GitID: gsUserResp.GitID,
+ Password: gsUserResp.Password,
+ }
+ err = c.userStore.Create(ctx, user, namespace)
+ if err != nil {
+ newError := fmt.Errorf("failed to create user,error:%w", err)
+ return database.User{}, newError
+ }
+
+ return *user, err
+}
+
+func (c *multiSyncComponentImpl) getUser(ctx context.Context, userName string) (database.User, error) {
+ return c.userStore.FindByUsername(ctx, userName)
+}
+
+func (c *multiSyncComponentImpl) createLocalSyncVersion(ctx context.Context, v types.SyncVersion) error {
+ syncVersion := database.SyncVersion{
+ Version: v.Version,
+ SourceID: v.SourceID,
+ RepoPath: v.RepoPath,
+ RepoType: v.RepoType,
+ LastModifiedAt: v.LastModifyTime,
+ ChangeLog: v.ChangeLog,
+ }
+ err := c.syncVersionStore.Create(ctx, &syncVersion)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "regexp"
+ "strings"
+ "sync"
+
+ "gopkg.in/yaml.v3"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/llm"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+var (
+ UserRole string = "user"
+ SystemRole string = "system"
+ AssistantRole string = "assistant"
+)
+
+type promptComponentImpl struct {
+ config *config.Config
+ userStore database.UserStore
+ userLikeStore database.UserLikesStore
+ userSvcClient rpc.UserSvcClient
+ promptConvStore database.PromptConversationStore
+ promptPrefixStore database.PromptPrefixStore
+ llmConfigStore database.LLMConfigStore
+ promptStore database.PromptStore
+ repoStore database.RepoStore
+ repoComponent RepoComponent
+ gitServer gitserver.GitServer
+ namespaceStore database.NamespaceStore
+ llmClient *llm.Client
+ maxPromptFS int64
+}
+
+type PromptComponent interface {
+ ListPrompt(ctx context.Context, req types.PromptReq) ([]PromptOutput, error)
+ GetPrompt(ctx context.Context, req types.PromptReq) (*PromptOutput, error)
+ ParseJsonFile(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (*PromptOutput, error)
+ CreatePrompt(ctx context.Context, req types.PromptReq, body *CreatePromptReq) (*Prompt, error)
+ UpdatePrompt(ctx context.Context, req types.PromptReq, body *UpdatePromptReq) (*Prompt, error)
+ DeletePrompt(ctx context.Context, req types.PromptReq) error
+ NewConversation(ctx context.Context, req types.ConversationTitleReq) (*database.PromptConversation, error)
+ ListConversationsByUserID(ctx context.Context, currentUser string) ([]database.PromptConversation, error)
+ GetConversation(ctx context.Context, req types.ConversationReq) (*database.PromptConversation, error)
+ SubmitMessage(ctx context.Context, req types.ConversationReq) (<-chan string, error)
+ SaveGeneratedText(ctx context.Context, req types.Conversation) (*database.PromptConversationMessage, error)
+ RemoveConversation(ctx context.Context, req types.ConversationReq) error
+ UpdateConversation(ctx context.Context, req types.ConversationTitleReq) (*database.PromptConversation, error)
+ LikeConversationMessage(ctx context.Context, req types.ConversationMessageReq) error
+ HateConversationMessage(ctx context.Context, req types.ConversationMessageReq) error
+ SetRelationModels(ctx context.Context, req types.RelationModels) error
+ AddRelationModel(ctx context.Context, req types.RelationModel) error
+ DelRelationModel(ctx context.Context, req types.RelationModel) error
+ CreatePromptRepo(ctx context.Context, req *types.CreatePromptRepoReq) (*types.PromptRes, error)
+ IndexPromptRepo(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.PromptRes, int, error)
+ UpdatePromptRepo(ctx context.Context, req *types.UpdatePromptRepoReq) (*types.PromptRes, error)
+ RemoveRepo(ctx context.Context, namespace, name, currentUser string) error
+ Show(ctx context.Context, namespace, name, currentUser string) (*types.PromptRes, error)
+ Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error)
+ OrgPrompts(ctx context.Context, req *types.OrgPromptsReq) ([]types.PromptRes, int, error)
+}
+
+func NewPromptComponent(cfg *config.Config) (PromptComponent, error) {
+ r, err := NewRepoComponentImpl(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create repo component,cause:%w", err)
+ }
+ gs, err := git.NewGitServer(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server,cause:%w", err)
+ }
+ usc := rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", cfg.User.Host, cfg.User.Port),
+ rpc.AuthWithApiKey(cfg.APIToken))
+ return &promptComponentImpl{
+ config: cfg,
+ userStore: database.NewUserStore(),
+ userLikeStore: database.NewUserLikesStore(),
+ userSvcClient: usc,
+ promptConvStore: database.NewPromptConversationStore(),
+ promptPrefixStore: database.NewPromptPrefixStore(),
+ llmConfigStore: database.NewLLMConfigStore(),
+ promptStore: database.NewPromptStore(),
+ llmClient: llm.NewClient(),
+ repoStore: database.NewRepoStore(),
+ repoComponent: r,
+ gitServer: gs,
+ maxPromptFS: cfg.Dataset.PromptMaxJsonlFileSize,
+ namespaceStore: database.NewNamespaceStore(),
+ }, nil
+}
+
+func (c *promptComponentImpl) ListPrompt(ctx context.Context, req types.PromptReq) ([]PromptOutput, error) {
+ r, err := c.repoStore.FindByPath(ctx, types.PromptRepo, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find prompt set, error: %w", err)
+ }
+
+ slog.Debug("ListPrompt check user permission begin")
+ allow, err := c.repoComponent.AllowReadAccessRepo(ctx, r, req.CurrentUser)
+ slog.Debug("ListPrompt check user permission end")
+ if err != nil {
+ return nil, fmt.Errorf("failed to check prompt set permission, error: %w", err)
+ }
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ slog.Debug("ListPrompt get repo file tree begin")
+ tree, err := GetFilePathObjects(req.Namespace, req.Name, "", types.PromptRepo, "", c.gitServer.GetRepoFileTree)
+ slog.Debug("ListPrompt get repo file tree end")
+ if err != nil {
+ return nil, fmt.Errorf("failed to get repo file tree, error: %w", err)
+ }
+ if tree == nil {
+ return nil, fmt.Errorf("failed to find any files")
+ }
+ var prompts []PromptOutput
+ wg := &sync.WaitGroup{}
+ chPrompts := make(chan *PromptOutput, len(tree))
+ done := make(chan struct{}, 1)
+
+ go func() {
+ for p := range chPrompts {
+ prompts = append(prompts, *p)
+ }
+ done <- struct{}{}
+ }()
+
+ for _, file := range tree {
+ if file.Lfs || file.Size > c.maxPromptFS {
+ slog.Warn("ListPromp skip large prompt file", slog.Any("filePath", file.Path), slog.Int64("size", file.Size))
+ continue
+ }
+ if !strings.HasSuffix(strings.ToLower(file.Path), ".jsonl") {
+ continue
+ }
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: file.Path,
+ RepoType: types.PromptRepo,
+ }
+
+ wg.Add(1)
+ go func(req gitserver.GetRepoInfoByPathReq) {
+ slog.Debug("ListPrompt parse prompt file begin", slog.String("file", req.Path))
+ p, err := c.ParseJsonFile(ctx, getFileContentReq)
+ if err != nil {
+ slog.Warn("fail to parse jsonl file", slog.Any("getFileContentReq", getFileContentReq), slog.Any("error", err))
+ }
+ slog.Debug("ListPrompt parse prompt file end", slog.String("file", req.Path))
+ chPrompts <- p
+ wg.Done()
+ }(getFileContentReq)
+ }
+
+ wg.Wait()
+ close(chPrompts)
+ <-done
+
+ return prompts, nil
+}
+
+func (c *promptComponentImpl) GetPrompt(ctx context.Context, req types.PromptReq) (*PromptOutput, error) {
+ r, err := c.repoStore.FindByPath(ctx, types.PromptRepo, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find prompt repo, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, req.CurrentUser, r)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: req.Path,
+ RepoType: types.PromptRepo,
+ }
+ p, err := c.ParseJsonFile(ctx, getFileContentReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse jsonl %s, error: %w", req.Path, err)
+ }
+ p.CanWrite = permission.CanWrite
+ p.CanManage = permission.CanAdmin
+ return p, nil
+}
+
+func (c *promptComponentImpl) ParseJsonFile(ctx context.Context, req gitserver.GetRepoInfoByPathReq) (*PromptOutput, error) {
+ f, err := c.gitServer.GetRepoFileContents(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get %s contents, cause:%w", req.Path, err)
+ }
+ decodedContent, err := base64.StdEncoding.DecodeString(f.Content)
+ if err != nil {
+ return nil, fmt.Errorf("failed to base64 decode %s contents, cause:%w", req.Path, err)
+ }
+ var prompt Prompt
+ err = yaml.Unmarshal(decodedContent, &prompt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to Unmarshal %s contents, cause: %w, decodedContent: %v", req.Path, err, string(decodedContent))
+ }
+ if len(prompt.Title) < 1 {
+ prompt.Title = f.Name
+ }
+ po := PromptOutput{
+ Prompt: prompt,
+ FilePath: req.Path,
+ }
+ return &po, nil
+}
+
+func (c *promptComponentImpl) CreatePrompt(ctx context.Context, req types.PromptReq, body *CreatePromptReq) (*Prompt, error) {
+ u, err := c.checkPromptRepoPermission(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("user do not allowed create prompt")
+ }
+ req.Path = fmt.Sprintf("%s.jsonl", body.Title)
+ exist, _ := c.checkFileExist(ctx, req)
+ if exist {
+ return nil, fmt.Errorf("prompt %s already exists", req.Path)
+ }
+ // generate json format string
+ promptJson, err := json.Marshal(body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to convert prompt to JSON, cause: %w", err)
+ }
+ promptJsonStr := base64.StdEncoding.EncodeToString(promptJson)
+
+ fileReq := types.CreateFileReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Branch: types.MainBranch,
+ FilePath: req.Path,
+ Content: promptJsonStr,
+ RepoType: types.PromptRepo,
+ CurrentUser: req.CurrentUser,
+ Username: req.CurrentUser,
+ Email: u.Email,
+ Message: fmt.Sprintf("create prompt %s", req.Path),
+ }
+ _, err = c.repoComponent.CreateFile(ctx, &fileReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create prompt file %s, cause: %w", req.Path, err)
+ }
+ return &body.Prompt, nil
+}
+
+func (c *promptComponentImpl) UpdatePrompt(ctx context.Context, req types.PromptReq, body *UpdatePromptReq) (*Prompt, error) {
+ u, err := c.checkPromptRepoPermission(ctx, req)
+ if err != nil {
+ return nil, fmt.Errorf("user do not allowed update prompt")
+ }
+ if !strings.HasSuffix(req.Path, ".jsonl") {
+ return nil, fmt.Errorf("prompt name must be end with .jsonl")
+ }
+ exist, _ := c.checkFileExist(ctx, req)
+ if !exist {
+ return nil, fmt.Errorf("prompt %s does not exist", req.Path)
+ }
+ promptJson, err := json.Marshal(body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to convert prompt to JSON, cause: %w", err)
+ }
+ promptJsonStr := base64.StdEncoding.EncodeToString(promptJson)
+
+ fileReq := types.UpdateFileReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Branch: types.MainBranch,
+ FilePath: req.Path,
+ Content: promptJsonStr,
+ RepoType: types.PromptRepo,
+ CurrentUser: req.CurrentUser,
+ Username: req.CurrentUser,
+ Email: u.Email,
+ Message: fmt.Sprintf("update prompt %s", req.Path),
+ }
+ _, err = c.repoComponent.UpdateFile(ctx, &fileReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update prompt file %s, cause: %w", req.Path, err)
+ }
+ return &body.Prompt, nil
+}
+
+func (c *promptComponentImpl) DeletePrompt(ctx context.Context, req types.PromptReq) error {
+ u, err := c.checkPromptRepoPermission(ctx, req)
+ if err != nil {
+ return fmt.Errorf("user do not allowed delete prompt")
+ }
+ if !strings.HasSuffix(req.Path, ".jsonl") {
+ return fmt.Errorf("prompt name must be end with .jsonl")
+ }
+
+ fileReq := types.DeleteFileReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Branch: types.MainBranch,
+ FilePath: req.Path,
+ Content: "",
+ RepoType: types.PromptRepo,
+ CurrentUser: req.CurrentUser,
+ Username: req.CurrentUser,
+ Email: u.Email,
+ Message: fmt.Sprintf("delete prompt %s", req.Path),
+ OriginPath: "",
+ }
+
+ _, err = c.repoComponent.DeleteFile(ctx, &fileReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete prompt %s, cause: %w", req.Path, err)
+ }
+ return nil
+}
+
+func (c *promptComponentImpl) checkFileExist(ctx context.Context, req types.PromptReq) (bool, error) {
+ getFileRawReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: req.Path,
+ RepoType: types.PromptRepo,
+ }
+ _, err := c.gitServer.GetRepoFileRaw(ctx, getFileRawReq)
+ if err != nil {
+ return false, fmt.Errorf("failed to get prompt repository %s/%s file %s raw, error: %w", req.Namespace, req.Name, req.Path, err)
+ }
+ return true, nil
+}
+
+func (c *promptComponentImpl) checkPromptRepoPermission(ctx context.Context, req types.PromptReq) (*database.User, error) {
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, errors.New("namespace does not exist")
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ if !user.CanAdmin() {
+ if namespace.NamespaceType == database.OrgNamespace {
+ canWrite, err := c.repoComponent.CheckCurrentUserPermission(ctx, req.CurrentUser, req.Namespace, membership.RoleWrite)
+ if err != nil {
+ return nil, err
+ }
+ if !canWrite {
+ return nil, errors.New("user do not have permission to update repo in this organization")
+ }
+ } else {
+ if namespace.Path != user.Username {
+ return nil, errors.New("user do not have permission to update repo in this namespace")
+ }
+ }
+ }
+ return &user, nil
+}
+
+func (c *promptComponentImpl) NewConversation(ctx context.Context, req types.ConversationTitleReq) (*database.PromptConversation, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ conversation := database.PromptConversation{
+ UserID: user.ID,
+ ConversationID: req.Uuid,
+ Title: req.Title,
+ }
+
+ err = c.promptConvStore.CreateConversation(ctx, conversation)
+ if err != nil {
+ return nil, fmt.Errorf("new conversation error: %w", err)
+ }
+
+ return &conversation, nil
+}
+
+func (c *promptComponentImpl) ListConversationsByUserID(ctx context.Context, currentUser string) ([]database.PromptConversation, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ conversations, err := c.promptConvStore.FindConversationsByUserID(ctx, user.ID)
+ if err != nil {
+ return nil, fmt.Errorf("find conversations by user %s error: %w", currentUser, err)
+ }
+ return conversations, nil
+}
+
+func (c *promptComponentImpl) GetConversation(ctx context.Context, req types.ConversationReq) (*database.PromptConversation, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ conversation, err := c.promptConvStore.GetConversationByID(ctx, user.ID, req.Uuid, true)
+ if err != nil {
+ return nil, fmt.Errorf("get conversation by id %s error: %w", req.Uuid, err)
+ }
+ return conversation, nil
+}
+
+func (c *promptComponentImpl) SubmitMessage(ctx context.Context, req types.ConversationReq) (<-chan string, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ _, err = c.promptConvStore.GetConversationByID(ctx, user.ID, req.Uuid, false)
+ if err != nil {
+ return nil, fmt.Errorf("invalid conversation by uuid %s error: %w", req.Uuid, err)
+ }
+
+ reqMsg := database.PromptConversationMessage{
+ ConversationID: req.Uuid,
+ Role: UserRole,
+ Content: req.Message,
+ }
+ _, err = c.promptConvStore.SaveConversationMessage(ctx, reqMsg)
+ if err != nil {
+ return nil, fmt.Errorf("save user prompt input error: %w", err)
+ }
+
+ llmConfig, err := c.llmConfigStore.GetOptimization(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("get llm config error: %w", err)
+ }
+ slog.Debug("use llm", slog.Any("llmConfig", llmConfig))
+ var headers map[string]string
+ err = json.Unmarshal([]byte(llmConfig.AuthHeader), &headers)
+ if err != nil {
+ return nil, fmt.Errorf("parse llm config header error: %w", err)
+ }
+
+ promptPrefix := ""
+ prefix, err := c.promptPrefixStore.Get(ctx)
+ if err != nil {
+ slog.Warn("fail to find prompt prefix", slog.Any("err", err))
+ } else {
+ chs := isChinese(reqMsg.Content)
+ if chs {
+ promptPrefix = prefix.ZH
+ } else {
+ promptPrefix = prefix.EN
+ }
+ }
+
+ reqData := types.LLMReqBody{
+ Model: llmConfig.ModelName,
+ Messages: []types.LLMMessage{
+ {Role: SystemRole, Content: promptPrefix},
+ {Role: UserRole, Content: reqMsg.Content},
+ },
+ Stream: true,
+ Temperature: 0.2,
+ }
+ if req.Temperature != nil {
+ reqData.Temperature = *req.Temperature
+ }
+
+ slog.Debug("llm request", slog.Any("reqData", reqData))
+ ch, err := c.llmClient.Chat(ctx, llmConfig.ApiEndpoint, headers, reqData)
+ if err != nil {
+ return nil, fmt.Errorf("call llm error: %w", err)
+ }
+ return ch, nil
+}
+
+func (c *promptComponentImpl) SaveGeneratedText(ctx context.Context, req types.Conversation) (*database.PromptConversationMessage, error) {
+ respMsg := database.PromptConversationMessage{
+ ConversationID: req.Uuid,
+ Role: AssistantRole,
+ Content: req.Message,
+ }
+ msg, err := c.promptConvStore.SaveConversationMessage(ctx, respMsg)
+ if err != nil {
+ return nil, fmt.Errorf("save system generated response error: %w", err)
+ }
+ return msg, nil
+}
+
+func (c *promptComponentImpl) RemoveConversation(ctx context.Context, req types.ConversationReq) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return errors.New("user does not exist")
+ }
+
+ err = c.promptConvStore.DeleteConversationsByID(ctx, user.ID, req.Uuid)
+ if err != nil {
+ return fmt.Errorf("remove conversation error: %w", err)
+ }
+ return nil
+}
+
+func (c *promptComponentImpl) UpdateConversation(ctx context.Context, req types.ConversationTitleReq) (*database.PromptConversation, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ err = c.promptConvStore.UpdateConversation(ctx, database.PromptConversation{
+ UserID: user.ID,
+ ConversationID: req.Uuid,
+ Title: req.Title,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("update conversation title error: %w", err)
+ }
+
+ resp, err := c.promptConvStore.GetConversationByID(ctx, user.ID, req.Uuid, false)
+ if err != nil {
+ return nil, fmt.Errorf("invalid conversation by uuid %s error: %w", req.Uuid, err)
+ }
+ return resp, nil
+}
+
+func (c *promptComponentImpl) LikeConversationMessage(ctx context.Context, req types.ConversationMessageReq) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return errors.New("user does not exist")
+ }
+ _, err = c.promptConvStore.GetConversationByID(ctx, user.ID, req.Uuid, false)
+ if err != nil {
+ return fmt.Errorf("invalid conversation by uuid %s error: %w", req.Uuid, err)
+ }
+ err = c.promptConvStore.LikeMessageByID(ctx, req.Id)
+ if err != nil {
+ return fmt.Errorf("update like message by id %d error: %w", req.Id, err)
+ }
+ return nil
+}
+
+func (c *promptComponentImpl) HateConversationMessage(ctx context.Context, req types.ConversationMessageReq) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return errors.New("user does not exist")
+ }
+ _, err = c.promptConvStore.GetConversationByID(ctx, user.ID, req.Uuid, false)
+ if err != nil {
+ return fmt.Errorf("invalid conversation by uuid %s error: %w", req.Uuid, err)
+ }
+ err = c.promptConvStore.HateMessageByID(ctx, req.Id)
+ if err != nil {
+ return fmt.Errorf("update hate message by id %d error: %w", req.Id, err)
+ }
+ return nil
+}
+
+func isChinese(s string) bool {
+ re := regexp.MustCompile(`[\p{Han}]`)
+ return re.MatchString(s)
+}
+
+func (c *promptComponentImpl) SetRelationModels(ctx context.Context, req types.RelationModels) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist, %w", err)
+ }
+
+ repo, err := c.repoStore.FindByPath(ctx, types.PromptRepo, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find prompt, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+
+ if !permission.CanWrite {
+ return fmt.Errorf("user %s do not allow to set relation models", req.CurrentUser)
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: REPOCARD_FILENAME,
+ RepoType: types.PromptRepo,
+ }
+ metaMap, splits, err := GetMetaMapFromReadMe(c.gitServer, getFileContentReq)
+ if err != nil {
+ return fmt.Errorf("failed parse meta from readme, cause: %w", err)
+ }
+ metaMap["models"] = req.Models
+ output, err := GetOutputForReadme(metaMap, splits)
+ if err != nil {
+ return fmt.Errorf("failed generate output for readme, cause: %w", err)
+ }
+
+ var readmeReq types.UpdateFileReq
+ readmeReq.Branch = types.MainBranch
+ readmeReq.Message = "update model relation tags"
+ readmeReq.FilePath = REPOCARD_FILENAME
+ readmeReq.RepoType = types.PromptRepo
+ readmeReq.Namespace = req.Namespace
+ readmeReq.Name = req.Name
+ readmeReq.Username = req.CurrentUser
+ readmeReq.Email = user.Email
+ readmeReq.Content = base64.StdEncoding.EncodeToString([]byte(output))
+
+ err = c.gitServer.UpdateRepoFile(&readmeReq)
+ if err != nil {
+ return fmt.Errorf("failed to set models tag to %s file, cause: %w", readmeReq.FilePath, err)
+ }
+
+ return nil
+}
+
+func GetMetaMapFromReadMe(git gitserver.GitServer, getFileContentReq gitserver.GetRepoInfoByPathReq) (map[string]any, []string, error) {
+ f, err := git.GetRepoFileContents(context.Background(), getFileContentReq)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to get readme.md contents, cause:%w", err)
+ }
+ decodedContent, err := base64.StdEncoding.DecodeString(f.Content)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to base64 decode readme.md contents, cause:%w", err)
+ }
+ decodedContentStr := string(decodedContent)
+ // slog.Info("get prompt readme", slog.Any("decodedContentStr", decodedContentStr))
+
+ splits := strings.Split(decodedContentStr, "---")
+ // slog.Info("split readme", slog.Any("len(splits)", len(splits)), slog.Any("splits", splits))
+
+ metaMap := make(map[string]any)
+ if len(splits) > 1 {
+ meta := splits[1]
+ //parse yaml string
+ err := yaml.Unmarshal([]byte(meta), metaMap)
+ if err != nil {
+ return nil, nil, fmt.Errorf("error unmarshall meta for prompt, cause: %w", err)
+ }
+ }
+ return metaMap, splits, nil
+}
+
+func GetOutputForReadme(metaMap map[string]any, splits []string) (string, error) {
+ yamlData, err := yaml.Marshal(metaMap)
+ if err != nil {
+ return "", fmt.Errorf("failed to marshal metaMap to YAML, cause: %w", err)
+ }
+ metaOutput := strings.Join([]string{"---", string(yamlData), "---"}, "\n")
+
+ output := ""
+ if len(splits) == 0 {
+ output = metaOutput
+ } else if len(splits) == 1 {
+ output = strings.Join([]string{metaOutput, splits[0]}, "\n")
+ } else {
+ splits[1] = metaOutput
+ output = strings.Join(splits, "")
+ }
+ // slog.Debug("update prompt readme", slog.Any("output", output))
+ return output, nil
+}
+
+func (c *promptComponentImpl) AddRelationModel(ctx context.Context, req types.RelationModel) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist, %w", err)
+ }
+
+ if !user.CanAdmin() {
+ return fmt.Errorf("only admin was allowed to set models for prompt")
+ }
+
+ _, err = c.repoStore.FindByPath(ctx, types.PromptRepo, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find prompt dataset, error: %w", err)
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: REPOCARD_FILENAME,
+ RepoType: types.PromptRepo,
+ }
+ metaMap, splits, err := GetMetaMapFromReadMe(c.gitServer, getFileContentReq)
+ if err != nil {
+ return fmt.Errorf("failed parse meta from readme, cause: %w", err)
+ }
+ models, ok := metaMap["models"]
+ if !ok {
+ models = []string{req.Model}
+ } else {
+ models = append(models.([]interface{}), req.Model)
+ }
+ metaMap["models"] = models
+ output, err := GetOutputForReadme(metaMap, splits)
+ if err != nil {
+ return fmt.Errorf("failed generate output for readme, cause: %w", err)
+ }
+
+ var readmeReq types.UpdateFileReq
+ readmeReq.Branch = types.MainBranch
+ readmeReq.Message = "add relation model"
+ readmeReq.FilePath = REPOCARD_FILENAME
+ readmeReq.RepoType = types.PromptRepo
+ readmeReq.Namespace = req.Namespace
+ readmeReq.Name = req.Name
+ readmeReq.Username = req.CurrentUser
+ readmeReq.Email = user.Email
+ readmeReq.Content = base64.StdEncoding.EncodeToString([]byte(output))
+
+ err = c.gitServer.UpdateRepoFile(&readmeReq)
+ if err != nil {
+ return fmt.Errorf("failed to add model tag to %s file, cause: %w", readmeReq.FilePath, err)
+ }
+
+ return nil
+}
+
+func (c *promptComponentImpl) DelRelationModel(ctx context.Context, req types.RelationModel) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("user does not exist, %w", err)
+ }
+
+ if !user.CanAdmin() {
+ return fmt.Errorf("only admin was allowed to delete model for prompt")
+ }
+
+ _, err = c.repoStore.FindByPath(ctx, types.PromptRepo, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find prompt, error: %w", err)
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: types.MainBranch,
+ Path: REPOCARD_FILENAME,
+ RepoType: types.PromptRepo,
+ }
+ metaMap, splits, err := GetMetaMapFromReadMe(c.gitServer, getFileContentReq)
+ if err != nil {
+ return fmt.Errorf("failed parse meta from readme, cause: %w", err)
+ }
+ models, ok := metaMap["models"]
+ if !ok {
+ return nil
+ } else {
+ var newModels []string
+ for _, v := range models.([]interface{}) {
+ if v.(string) != req.Model {
+ newModels = append(newModels, v.(string))
+ }
+ }
+ metaMap["models"] = newModels
+ }
+ output, err := GetOutputForReadme(metaMap, splits)
+ if err != nil {
+ return fmt.Errorf("failed generate output for readme, cause: %w", err)
+ }
+
+ var readmeReq types.UpdateFileReq
+ readmeReq.Branch = types.MainBranch
+ readmeReq.Message = "delete relation model"
+ readmeReq.FilePath = REPOCARD_FILENAME
+ readmeReq.RepoType = types.PromptRepo
+ readmeReq.Namespace = req.Namespace
+ readmeReq.Name = req.Name
+ readmeReq.Username = req.CurrentUser
+ readmeReq.Email = user.Email
+ readmeReq.Content = base64.StdEncoding.EncodeToString([]byte(output))
+
+ err = c.gitServer.UpdateRepoFile(&readmeReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete model tag to %s file, cause: %w", readmeReq.FilePath, err)
+ }
+
+ return nil
+}
+
+func (c *promptComponentImpl) CreatePromptRepo(ctx context.Context, req *types.CreatePromptRepoReq) (*types.PromptRes, error) {
+ var (
+ nickname string
+ tags []types.RepoTag
+ )
+
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, errors.New("namespace does not exist")
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ if namespace.NamespaceType == database.OrgNamespace {
+ canWrite, err := c.repoComponent.CheckCurrentUserPermission(ctx, req.Username, req.Namespace, membership.RoleWrite)
+ if err != nil {
+ return nil, err
+ }
+ if !canWrite {
+ return nil, errors.New("users do not have permission to create prompt in this organization")
+ }
+ } else {
+ if namespace.Path != user.Username {
+ return nil, errors.New("users do not have permission to create prompt in this namespace")
+ }
+ }
+ }
+
+ if req.Nickname != "" {
+ nickname = req.Nickname
+ } else {
+ nickname = req.Name
+ }
+
+ if req.DefaultBranch == "" {
+ req.DefaultBranch = types.MainBranch
+ }
+
+ req.RepoType = types.PromptRepo
+ req.Readme = generateReadmeData(req.License)
+ req.Nickname = nickname
+ _, dbRepo, err := c.repoComponent.CreateRepo(ctx, req.CreateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ dbPrompt := database.Prompt{
+ Repository: dbRepo,
+ RepositoryID: dbRepo.ID,
+ }
+
+ prompt, err := c.promptStore.Create(ctx, dbPrompt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create database prompt, cause: %w", err)
+ }
+
+ // Create README.md file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: user.Username,
+ Email: user.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: req.Readme,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: readmeFileName,
+ }, types.PromptRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create prompt repo README.md file, cause: %w", err)
+ }
+
+ // Create .gitattributes file
+ err = c.gitServer.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: user.Username,
+ Email: user.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: datasetGitattributesContent,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: gitattributesFileName,
+ }, types.PromptRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create .gitattributes file, cause: %w", err)
+ }
+
+ for _, tag := range prompt.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ resPrompt := &types.PromptRes{
+ ID: prompt.ID,
+ Name: prompt.Repository.Name,
+ Nickname: prompt.Repository.Nickname,
+ Description: prompt.Repository.Description,
+ Likes: prompt.Repository.Likes,
+ Downloads: prompt.Repository.DownloadCount,
+ Path: prompt.Repository.Path,
+ RepositoryID: prompt.RepositoryID,
+ Repository: common.BuildCloneInfo(c.config, prompt.Repository),
+ Private: prompt.Repository.Private,
+ User: types.User{
+ Username: user.Username,
+ Nickname: user.NickName,
+ Email: user.Email,
+ },
+ Tags: tags,
+ CreatedAt: prompt.CreatedAt,
+ UpdatedAt: prompt.UpdatedAt,
+ }
+
+ return resPrompt, nil
+}
+
+func (c *promptComponentImpl) IndexPromptRepo(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.PromptRes, int, error) {
+ var (
+ err error
+ resPrompts []types.PromptRes
+ )
+ repos, total, err := c.repoComponent.PublicToUser(ctx, types.PromptRepo, filter.Username, filter, per, page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public prompt repos,error:%w", err)
+ return nil, 0, newError
+ }
+ var repoIDs []int64
+ for _, repo := range repos {
+ repoIDs = append(repoIDs, repo.ID)
+ }
+ prompts, err := c.promptStore.ByRepoIDs(ctx, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get prompts by repo ids,error:%w", err)
+ return nil, 0, newError
+ }
+
+ //loop through repos to keep the repos in sort order
+ for _, repo := range repos {
+ var prompt *database.Prompt
+ for _, d := range prompts {
+ if repo.ID == d.RepositoryID {
+ prompt = &d
+ break
+ }
+ }
+ if prompt == nil {
+ continue
+ }
+ var tags []types.RepoTag
+ for _, tag := range repo.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ resPrompts = append(resPrompts, types.PromptRes{
+ ID: prompt.ID,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ Likes: repo.Likes,
+ Downloads: repo.DownloadCount,
+ Path: repo.Path,
+ RepositoryID: repo.ID,
+ Private: repo.Private,
+ Tags: tags,
+ CreatedAt: prompt.CreatedAt,
+ UpdatedAt: repo.UpdatedAt,
+ Source: repo.Source,
+ SyncStatus: repo.SyncStatus,
+ License: repo.License,
+ Repository: common.BuildCloneInfo(c.config, prompt.Repository),
+
+ User: types.User{
+ Username: prompt.Repository.User.Username,
+ Nickname: prompt.Repository.User.NickName,
+ Email: prompt.Repository.User.Email,
+ Avatar: prompt.Repository.User.Avatar,
+ },
+ })
+ }
+
+ return resPrompts, total, nil
+}
+
+func (c *promptComponentImpl) UpdatePromptRepo(ctx context.Context, req *types.UpdatePromptRepoReq) (*types.PromptRes, error) {
+ req.RepoType = types.PromptRepo
+ dbRepo, err := c.repoComponent.UpdateRepo(ctx, req.UpdateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ prompt, err := c.promptStore.ByRepoID(ctx, dbRepo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find prompt, error: %w", err)
+ }
+
+ // update times of prompt repo
+ err = c.promptStore.Update(ctx, *prompt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update prompt, error: %w", err)
+ }
+
+ resPrompt := &types.PromptRes{
+ ID: prompt.ID,
+ Name: dbRepo.Name,
+ Nickname: dbRepo.Nickname,
+ Description: dbRepo.Description,
+ Likes: dbRepo.Likes,
+ Downloads: dbRepo.DownloadCount,
+ Path: dbRepo.Path,
+ RepositoryID: dbRepo.ID,
+ Private: dbRepo.Private,
+ CreatedAt: prompt.CreatedAt,
+ UpdatedAt: prompt.UpdatedAt,
+ }
+
+ return resPrompt, nil
+}
+
+func (c *promptComponentImpl) RemoveRepo(ctx context.Context, namespace, name, currentUser string) error {
+ prompt, err := c.promptStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find prompt, error: %w", err)
+ }
+
+ deleteDatabaseRepoReq := types.DeleteRepoReq{
+ Username: currentUser,
+ Namespace: namespace,
+ Name: name,
+ RepoType: types.PromptRepo,
+ }
+ _, err = c.repoComponent.DeleteRepo(ctx, deleteDatabaseRepoReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete repo of prompt, error: %w", err)
+ }
+
+ err = c.promptStore.Delete(ctx, *prompt)
+ if err != nil {
+ return fmt.Errorf("failed to delete database prompt, error: %w", err)
+ }
+ return nil
+}
+
+func (c *promptComponentImpl) Show(ctx context.Context, namespace, name, currentUser string) (*types.PromptRes, error) {
+ var tags []types.RepoTag
+ prompt, err := c.promptStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find prompt, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, currentUser, prompt.Repository)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ ns, err := c.repoComponent.GetNameSpaceInfo(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace info for prompt, error: %w", err)
+ }
+
+ for _, tag := range prompt.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+
+ likeExists, err := c.userLikeStore.IsExist(ctx, currentUser, prompt.Repository.ID)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user likes,error:%w", err)
+ return nil, newError
+ }
+
+ resPrompt := &types.PromptRes{
+ ID: prompt.ID,
+ Name: prompt.Repository.Name,
+ Nickname: prompt.Repository.Nickname,
+ Description: prompt.Repository.Description,
+ Likes: prompt.Repository.Likes,
+ Downloads: prompt.Repository.DownloadCount,
+ Path: prompt.Repository.Path,
+ RepositoryID: prompt.Repository.ID,
+ DefaultBranch: prompt.Repository.DefaultBranch,
+ Repository: common.BuildCloneInfo(c.config, prompt.Repository),
+ Tags: tags,
+ User: types.User{
+ Username: prompt.Repository.User.Username,
+ Nickname: prompt.Repository.User.NickName,
+ Email: prompt.Repository.User.Email,
+ Avatar: prompt.Repository.User.Avatar,
+ },
+ Private: prompt.Repository.Private,
+ CreatedAt: prompt.CreatedAt,
+ UpdatedAt: prompt.Repository.UpdatedAt,
+ UserLikes: likeExists,
+ Source: prompt.Repository.Source,
+ SyncStatus: prompt.Repository.SyncStatus,
+ License: prompt.Repository.License,
+ CanWrite: permission.CanWrite,
+ CanManage: permission.CanAdmin,
+ Namespace: ns,
+ }
+ if permission.CanAdmin {
+ resPrompt.SensitiveCheckStatus = prompt.Repository.SensitiveCheckStatus.String()
+ }
+
+ return resPrompt, nil
+}
+
+func (c *promptComponentImpl) Relations(ctx context.Context, namespace, name, currentUser string) (*types.Relations, error) {
+ prompt, err := c.promptStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find prompt repo, error: %w", err)
+ }
+
+ allow, _ := c.repoComponent.AllowReadAccessRepo(ctx, prompt.Repository, currentUser)
+ if !allow {
+ return nil, ErrUnauthorized
+ }
+
+ return c.getRelations(ctx, prompt.RepositoryID, currentUser)
+}
+
+func (c *promptComponentImpl) getRelations(ctx context.Context, repoID int64, currentUser string) (*types.Relations, error) {
+ res, err := c.repoComponent.RelatedRepos(ctx, repoID, currentUser)
+ if err != nil {
+ return nil, err
+ }
+ rels := new(types.Relations)
+ modelRepos := res[types.ModelRepo]
+ for _, repo := range modelRepos {
+ rels.Models = append(rels.Models, &types.Model{
+ Path: repo.Path,
+ Name: repo.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ UpdatedAt: repo.UpdatedAt,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ })
+ }
+
+ return rels, nil
+}
+
+type Prompt struct {
+ Title string `json:"title" binding:"required"`
+ Content string `json:"content" binding:"required"`
+ Language string `json:"language" binding:"required"`
+ Tags []string `json:"tags"`
+ Type string `json:"type"` // "text|image|video|audio"
+ Source string `json:"source"`
+ Author string `json:"author"`
+ Time string `json:"time"`
+ Copyright string `json:"copyright"`
+ Feedback []string `json:"feedback"`
+}
+
+type PromptOutput struct {
+ Prompt
+ FilePath string `json:"file_path"`
+ CanWrite bool `json:"can_write"`
+ CanManage bool `json:"can_manage"`
+}
+
+type CreatePromptReq struct {
+ Prompt
+}
+
+type UpdatePromptReq struct {
+ Prompt
+}
+
+var _ types.SensitiveRequestV2 = (*Prompt)(nil)
+
+func (req *Prompt) GetSensitiveFields() []types.SensitiveField {
+ var fields []types.SensitiveField
+ fields = append(fields, types.SensitiveField{
+ Name: "title",
+ Value: func() string {
+ return req.Title
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ })
+ fields = append(fields, types.SensitiveField{
+ Name: "content",
+ Value: func() string {
+ return req.Content
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ })
+ if len(req.Source) > 0 {
+ fields = append(fields, types.SensitiveField{
+ Name: "source",
+ Value: func() string {
+ return req.Source
+ },
+ Scenario: string(sensitive.ScenarioCommentDetection),
+ })
+ }
+ return fields
+}
+
+func (c *promptComponentImpl) OrgPrompts(ctx context.Context, req *types.OrgPromptsReq) ([]types.PromptRes, int, error) {
+ var resPrompts []types.PromptRes
+ var err error
+ r := membership.RoleUnknown
+ if req.CurrentUser != "" {
+ r, err = c.userSvcClient.GetMemberRole(ctx, req.Namespace, req.CurrentUser)
+ // log error, and treat user as unknown role in org
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Namespace), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ }
+ onlyPublic := !r.CanRead()
+ prompts, total, err := c.promptStore.ByOrgPath(ctx, req.Namespace, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user prompts,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range prompts {
+ resPrompts = append(resPrompts, types.PromptRes{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resPrompts, total, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/d5/tengo/v2"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+)
+
+type recomComponentImpl struct {
+ recomStore database.RecomStore
+ userStore database.UserStore
+ repoStore database.RepoStore
+ gitServer gitserver.GitServer
+}
+
+type RecomComponent interface {
+ SetOpWeight(ctx context.Context, repoID, weight int64, userName string) error
+ // loop through repositories and calculate the recom score of the repository
+ CalculateRecomScore(ctx context.Context)
+ CalcTotalScore(ctx context.Context, repo *database.Repository, weights map[string]string) float64
+}
+
+func NewRecomComponent(cfg *config.Config) (RecomComponent, error) {
+ gs, err := git.NewGitServer(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to init git server,%w", err)
+ }
+
+ return &recomComponentImpl{
+ recomStore: database.NewRecomStore(),
+ userStore: database.NewUserStore(),
+ repoStore: database.NewRepoStore(),
+ gitServer: gs,
+ }, nil
+}
+
+func (rc *recomComponentImpl) SetOpWeight(ctx context.Context, repoID, weight int64, userName string) error {
+ _, err := rc.repoStore.FindById(ctx, repoID)
+ if err != nil {
+ return fmt.Errorf("failed to find repository with id %d, err:%w", repoID, err)
+ }
+ u, err := rc.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return fmt.Errorf("failed to find user with name %s, err:%w", userName, err)
+ }
+ if !u.CanAdmin() {
+ return fmt.Errorf("user %s is not admin", userName)
+ }
+ return rc.recomStore.UpsetOpWeights(ctx, repoID, weight)
+}
+
+// loop through repositories and calculate the recom score of the repository
+func (rc *recomComponentImpl) CalculateRecomScore(ctx context.Context) {
+ weights, err := rc.loadWeights()
+ if err != nil {
+ slog.Error("Error loading weights", slog.Any("error", err))
+ return
+ }
+ repos, err := rc.repoStore.All(ctx)
+ if err != nil {
+ slog.Error("Error fetching repositories", slog.Any("error", err))
+ return
+ }
+ for _, repo := range repos {
+ repoID := repo.ID
+ score := rc.CalcTotalScore(ctx, repo, weights)
+ err := rc.recomStore.UpsertScore(ctx, repoID, score)
+ if err != nil {
+ slog.Error("Error updating recom score", slog.Int64("repo_id", repoID), slog.Float64("score", score),
+ slog.String("error", err.Error()))
+ }
+ }
+}
+
+func (rc *recomComponentImpl) CalcTotalScore(ctx context.Context, repo *database.Repository, weights map[string]string) float64 {
+ score := float64(0)
+
+ if freshness, ok := weights["freshness"]; ok {
+ score += rc.calcFreshnessScore(repo.CreatedAt, freshness)
+ }
+
+ if downloads, ok := weights["downloads"]; ok {
+ score += rc.calcDownloadsScore(repo.DownloadCount, downloads)
+ }
+
+ qualityScore, err := rc.calcQualityScore(ctx, repo)
+ if err != nil {
+ slog.Error("failed to calculate quality score", slog.Any("error", err))
+ } else {
+ score += qualityScore
+ }
+
+ return score
+}
+
+func (rc *recomComponentImpl) calcFreshnessScore(createdAt time.Time, weightExp string) float64 {
+ // TODO:cache compiled script
+ hours := time.Since(createdAt).Hours()
+ scriptFreshness := tengo.NewScript([]byte(weightExp))
+ _ = scriptFreshness.Add("score", 0.0)
+ _ = scriptFreshness.Add("hours", 0)
+ sc, err := scriptFreshness.Compile()
+ if err != nil {
+ panic(err)
+ }
+ _ = sc.Set("hours", hours)
+ err = sc.Run()
+ if err != nil {
+ panic(err)
+ }
+
+ return sc.Get("score").Float()
+}
+
+func (rc *recomComponentImpl) calcDownloadsScore(downloads int64, weightExp string) float64 {
+ // TODO:cache compiled script
+ scriptFreshness := tengo.NewScript([]byte(weightExp))
+ _ = scriptFreshness.Add("score", 0.0)
+ _ = scriptFreshness.Add("downloads", 0)
+ sc, err := scriptFreshness.Compile()
+ if err != nil {
+ panic(err)
+ }
+ _ = sc.Set("downloads", downloads)
+ err = sc.Run()
+ if err != nil {
+ panic(err)
+ }
+
+ return sc.Get("score").Float()
+}
+
+func (rc *recomComponentImpl) calcQualityScore(ctx context.Context, repo *database.Repository) (float64, error) {
+ score := 0.0
+ // get file counts from git server
+ namespace, name := repo.NamespaceAndName()
+ files, err := getFilePaths(namespace, name, "", repo.RepositoryType, "", rc.gitServer.GetRepoFileTree)
+ if err != nil {
+ return 0, fmt.Errorf("failed to get repo file tree,%w", err)
+ }
+ fileCount := len(files)
+ for _, f := range files {
+ if f == "README.md" {
+ fileCount--
+ }
+ if f == "LICENSE" {
+ fileCount--
+ }
+ if f == ".gitattributes" {
+ fileCount--
+ }
+ }
+
+ if fileCount >= 2 {
+ score += 300.0
+ }
+ return score, nil
+}
+
+func (rc *recomComponentImpl) loadWeights() (map[string]string, error) {
+ ctx := context.Background()
+ items, err := rc.recomStore.LoadWeights(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ weights := make(map[string]string)
+ for _, item := range items {
+ weights[item.Name] = item.WeightExp
+ }
+ return weights, nil
+}
+
+// func (rc *recomComponentImpl) loadOpWeights() (map[int64]int, error) {
+// ctx := context.Background()
+// items, err := rc.rs.LoadOpWeights(ctx)
+// if err != nil {
+// return nil, err
+// }
+
+// weights := make(map[int64]int)
+// for _, item := range items {
+// weights[item.RepositoryID] = item.Weight
+// }
+// return weights, nil
+// }
+
+
+
package component
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "database/sql"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "path"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "opencsg.com/csghub-server/builder/deploy"
+ deployStatus "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/git/mirrorserver"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/s3"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/mirror/cache"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+const (
+ ErrNotFoundMessage = "The target couldn't be found."
+ ErrGetContentsOrList = "GetContentsOrList"
+ AdminSecret = "gnuRYKce"
+ GitAttributesFileName = ".gitattributes"
+)
+
+type repoComponentImpl struct {
+ tagComponent TagComponent
+ userStore database.UserStore
+ orgStore database.OrgStore
+ namespaceStore database.NamespaceStore
+ repoStore database.RepoStore
+ repoFileStore database.RepoFileStore
+ repoRelationsStore database.RepoRelationsStore
+ mirrorStore database.MirrorStore
+ git gitserver.GitServer
+ s3Client s3.Client
+ userSvcClient rpc.UserSvcClient
+ lfsBucket string
+ userLikesStore database.UserLikesStore
+ mirrorServer mirrorserver.MirrorServer
+ runtimeFrameworksStore database.RuntimeFrameworksStore
+ deployTaskStore database.DeployTaskStore
+ deployer deploy.Deployer
+ publicRootDomain string
+ serverBaseUrl string
+ clusterInfoStore database.ClusterInfoStore
+ mirrorSourceStore database.MirrorSourceStore
+ tokenStore database.AccessTokenStore
+ repoRuntimeFrameworkStore database.RepositoriesRuntimeFrameworkStore
+ syncVersionStore database.SyncVersionStore
+ syncClientSettingStore database.SyncClientSettingStore
+ fileStore database.FileStore
+ config *config.Config
+ accountingComponent AccountingComponent
+ spaceResourceStore database.SpaceResourceStore
+ lfsMetaObjectStore database.LfsMetaObjectStore
+ userResourcesStore database.UserResourcesStore
+ recomStore database.RecomStore
+ mq queue.PriorityQueue
+ syncCache cache.Cache
+}
+
+type RepoComponent interface {
+ CreateRepo(ctx context.Context, req types.CreateRepoReq) (*gitserver.CreateRepoResp, *database.Repository, error)
+ UpdateRepo(ctx context.Context, req types.UpdateRepoReq) (*database.Repository, error)
+ DeleteRepo(ctx context.Context, req types.DeleteRepoReq) (*database.Repository, error)
+ // PublicToUser gets visible repos of the given user and user's orgs
+ PublicToUser(ctx context.Context, repoType types.RepositoryType, userName string, filter *types.RepoFilter, per, page int) (repos []*database.Repository, count int, err error)
+ CreateFile(ctx context.Context, req *types.CreateFileReq) (*types.CreateFileResp, error)
+ UpdateFile(ctx context.Context, req *types.UpdateFileReq) (*types.UpdateFileResp, error)
+ DeleteFile(ctx context.Context, req *types.DeleteFileReq) (*types.DeleteFileResp, error)
+ Commits(ctx context.Context, req *types.GetCommitsReq) ([]types.Commit, *types.RepoPageOpts, error)
+ LastCommit(ctx context.Context, req *types.GetCommitsReq) (*types.Commit, error)
+ FileRaw(ctx context.Context, req *types.GetFileReq) (string, error)
+ DownloadFile(ctx context.Context, req *types.GetFileReq, userName string) (io.ReadCloser, int64, string, error)
+ Branches(ctx context.Context, req *types.GetBranchesReq) ([]types.Branch, error)
+ Tags(ctx context.Context, req *types.GetTagsReq) ([]database.Tag, error)
+ UpdateTags(ctx context.Context, namespace, name string, repoType types.RepositoryType, category, currentUser string, tags []string) error
+ Tree(ctx context.Context, req *types.GetFileReq) ([]*types.File, error)
+ UploadFile(ctx context.Context, req *types.CreateFileReq) error
+ SDKListFiles(ctx context.Context, repoType types.RepositoryType, namespace, name, ref, userName string) (*types.SDKFiles, error)
+ IsLfs(ctx context.Context, req *types.GetFileReq) (bool, int64, error)
+ HeadDownloadFile(ctx context.Context, req *types.GetFileReq, userName string) (*types.File, *types.Commit, error)
+ SDKDownloadFile(ctx context.Context, req *types.GetFileReq, userName string) (io.ReadCloser, int64, string, error)
+ // UpdateDownloads increase clone download count for repo by given count
+ UpdateDownloads(ctx context.Context, req *types.UpdateDownloadsReq) error
+ // IncrDownloads increase the click download count for repo by 1
+ IncrDownloads(ctx context.Context, repoType types.RepositoryType, namespace, name string) error
+ FileInfo(ctx context.Context, req *types.GetFileReq) (*types.File, error)
+ AllowReadAccessRepo(ctx context.Context, repo *database.Repository, username string) (bool, error)
+ AllowReadAccess(ctx context.Context, repoType types.RepositoryType, namespace, name, username string) (bool, error)
+ AllowWriteAccess(ctx context.Context, repoType types.RepositoryType, namespace, name, username string) (bool, error)
+ AllowAdminAccess(ctx context.Context, repoType types.RepositoryType, namespace, name, username string) (bool, error)
+ GetCommitWithDiff(ctx context.Context, req *types.GetCommitsReq) (*types.CommitResponse, error)
+ CreateMirror(ctx context.Context, req types.CreateMirrorReq) (*database.Mirror, error)
+ MirrorFromSaas(ctx context.Context, namespace, name, currentUser string, repoType types.RepositoryType) error
+ GetMirror(ctx context.Context, req types.GetMirrorReq) (*database.Mirror, error)
+ UpdateMirror(ctx context.Context, req types.UpdateMirrorReq) (*database.Mirror, error)
+ DeleteMirror(ctx context.Context, req types.DeleteMirrorReq) error
+ // get runtime framework list with type
+ ListRuntimeFrameworkWithType(ctx context.Context, deployType int) ([]types.RuntimeFramework, error)
+ // get runtime framework list
+ ListRuntimeFramework(ctx context.Context, repoType types.RepositoryType, namespace, name string, deployType int) ([]types.RuntimeFramework, error)
+ CreateRuntimeFramework(ctx context.Context, req *types.RuntimeFrameworkReq) (*types.RuntimeFramework, error)
+ UpdateRuntimeFramework(ctx context.Context, id int64, req *types.RuntimeFrameworkReq) (*types.RuntimeFramework, error)
+ DeleteRuntimeFramework(ctx context.Context, id int64) error
+ ListDeploy(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string) ([]types.DeployRepo, error)
+ DeleteDeploy(ctx context.Context, delReq types.DeployActReq) error
+ DeployDetail(ctx context.Context, detailReq types.DeployActReq) (*types.DeployRepo, error)
+ DeployInstanceLogs(ctx context.Context, logReq types.DeployActReq) (*deploy.MultiLogReader, error)
+ // check access repo permission by repo id
+ AllowAccessByRepoID(ctx context.Context, repoID int64, username string) (bool, error)
+ // check access endpoint for rproxy
+ AllowAccessEndpoint(ctx context.Context, currentUser string, deploy *database.Deploy) (bool, error)
+ // check access deploy permission
+ AllowAccessDeploy(ctx context.Context, req types.DeployActReq) (bool, error)
+ DeployStop(ctx context.Context, stopReq types.DeployActReq) error
+ AllowReadAccessByDeployID(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string, deployID int64) (bool, error)
+ DeployStatus(ctx context.Context, repoType types.RepositoryType, namespace, name string, deployID int64) (string, string, []types.Instance, error)
+ GetDeployBySvcName(ctx context.Context, svcName string) (*database.Deploy, error)
+ SyncMirror(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string) error
+ MirrorProgress(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string) ([]types.LFSSyncProgressResp, error)
+ DeployUpdate(ctx context.Context, updateReq types.DeployActReq, req *types.DeployUpdateReq) error
+ DeployStart(ctx context.Context, startReq types.DeployActReq) error
+ AllFiles(ctx context.Context, req types.GetAllFilesReq) ([]*types.File, error)
+ GetUserRepoPermission(ctx context.Context, userName string, repo *database.Repository) (*types.UserRepoPermission, error)
+ CheckCurrentUserPermission(ctx context.Context, userName string, namespace string, role membership.Role) (bool, error)
+ GetNameSpaceInfo(ctx context.Context, path string) (*types.Namespace, error)
+ RelatedRepos(ctx context.Context, repoID int64, currentUser string) (map[types.RepositoryType][]*database.Repository, error)
+ VisiableToUser(ctx context.Context, repos []*database.Repository, currentUser string) ([]*database.Repository, error)
+ GenerateEndpoint(ctx context.Context, deploy *database.Deploy) (string, string)
+ IsAdminRole(user database.User) bool
+}
+
+func NewRepoComponentImpl(config *config.Config) (*repoComponentImpl, error) {
+ r, err := NewRepoComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return r.(*repoComponentImpl), nil
+}
+
+func NewRepoComponent(config *config.Config) (RepoComponent, error) {
+ c := &repoComponentImpl{}
+ c.namespaceStore = database.NewNamespaceStore()
+ c.userStore = database.NewUserStore()
+ c.orgStore = database.NewOrgStore()
+ c.repoStore = database.NewRepoStore()
+ c.repoFileStore = database.NewRepoFileStore()
+ c.repoRelationsStore = database.NewRepoRelationsStore()
+ c.userLikesStore = database.NewUserLikesStore()
+ c.mirrorStore = database.NewMirrorStore()
+ c.mirrorSourceStore = database.NewMirrorSourceStore()
+ c.tokenStore = database.NewAccessTokenStore()
+ c.syncVersionStore = database.NewSyncVersionStore()
+ c.syncClientSettingStore = database.NewSyncClientSettingStore()
+ c.fileStore = database.NewFileStore()
+ var err error
+ mq, err := queue.GetPriorityQueueInstance()
+ if err != nil {
+ return nil, fmt.Errorf("failed to get priority queue: %v", err)
+ }
+ c.mq = mq
+ c.git, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ if config.GitServer.Type == types.GitServerTypeGitea {
+ c.mirrorServer, err = git.NewMirrorServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git mirror server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ }
+ c.tagComponent, err = NewTagComponent(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create tag component,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.s3Client, err = s3.NewMinio(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to init s3 client for code,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.lfsBucket = config.S3.Bucket
+ c.userSvcClient = rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+ c.runtimeFrameworksStore = database.NewRuntimeFrameworksStore()
+ c.deployTaskStore = database.NewDeployTaskStore()
+ c.deployer = deploy.NewDeployer()
+ c.publicRootDomain = config.Space.PublicRootDomain
+ c.serverBaseUrl = config.APIServer.PublicDomain
+ c.clusterInfoStore = database.NewClusterInfoStore()
+ c.runtimeFrameworksStore = database.NewRuntimeFrameworksStore()
+ c.repoRuntimeFrameworkStore = database.NewRepositoriesRuntimeFramework()
+ c.accountingComponent, err = NewAccountingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.spaceResourceStore = database.NewSpaceResourceStore()
+ c.lfsMetaObjectStore = database.NewLfsMetaObjectStore()
+ c.userResourcesStore = database.NewUserResourcesStore()
+ c.recomStore = database.NewRecomStore()
+ c.config = config
+ c.syncCache, err = cache.NewCache(context.Background(), config)
+ if err != nil {
+ return nil, fmt.Errorf("initializing redis: %w", err)
+ }
+ return c, nil
+}
+
+func (c *repoComponentImpl) CreateRepo(ctx context.Context, req types.CreateRepoReq) (*gitserver.CreateRepoResp, *database.Repository, error) {
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, nil, errors.New("namespace does not exist")
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, nil, errors.New("user does not exist")
+ }
+
+ if user.Email == "" {
+ return nil, nil, fmt.Errorf("please set your email first")
+ }
+
+ if !user.CanAdmin() {
+ if namespace.NamespaceType == database.OrgNamespace {
+ canWrite, err := c.CheckCurrentUserPermission(ctx, req.Username, req.Namespace, membership.RoleWrite)
+ if err != nil {
+ return nil, nil, err
+ }
+ if !canWrite {
+ return nil, nil, fmt.Errorf("users do not have permission to create %s in this organization", req.RepoType)
+ }
+ } else {
+ if namespace.Path != user.Username {
+ return nil, nil, fmt.Errorf("users do not have permission to create %s in this namespace", req.RepoType)
+ }
+ }
+ }
+ if req.DefaultBranch == "" {
+ req.DefaultBranch = types.MainBranch
+ }
+
+ gitRepoReq := gitserver.CreateRepoReq{
+ Username: req.Username,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Nickname: req.Name,
+ License: req.License,
+ DefaultBranch: req.DefaultBranch,
+ // Readme: "Please introduce your space.",
+ Readme: req.Readme,
+ Private: req.Private,
+ RepoType: req.RepoType,
+ }
+ gitRepo, err := c.git.CreateRepo(ctx, gitRepoReq)
+ if err != nil {
+ slog.Error("fail to create repo in git ", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, nil, fmt.Errorf("fail to create repo in git, error: %w", err)
+ }
+
+ dbRepo := database.Repository{
+ UserID: user.ID,
+ Path: path.Join(req.Namespace, req.Name),
+ GitPath: gitRepo.GitPath,
+ Name: req.Name,
+ Nickname: req.Nickname,
+ Description: req.Description,
+ Private: req.Private,
+ License: req.License,
+ DefaultBranch: gitRepo.DefaultBranch,
+ RepositoryType: req.RepoType,
+ HTTPCloneURL: gitRepo.HttpCloneURL,
+ SSHCloneURL: gitRepo.SshCloneURL,
+ }
+ newDBRepo, err := c.repoStore.CreateRepo(ctx, dbRepo)
+ if err != nil {
+ return nil, nil, fmt.Errorf("fail to create database repo, error: %w", err)
+ }
+ newDBRepo.User = user
+
+ return gitRepo, newDBRepo, nil
+}
+
+func (c *repoComponentImpl) UpdateRepo(ctx context.Context, req types.UpdateRepoReq) (*database.Repository, error) {
+ repo, err := c.repoStore.Find(ctx, req.Namespace, string(req.RepoType), req.Name)
+ if err != nil {
+ return nil, errors.New("repository does not exist")
+ }
+
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, errors.New("namespace does not exist")
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ if !user.CanAdmin() {
+ if namespace.NamespaceType == database.OrgNamespace {
+ canWrite, err := c.CheckCurrentUserPermission(ctx, req.Username, req.Namespace, membership.RoleWrite)
+ if err != nil {
+ return nil, err
+ }
+ if !canWrite {
+ return nil, errors.New("users do not have permission to update repo in this organization")
+ }
+ } else {
+ if namespace.Path != user.Username {
+ return nil, errors.New("users do not have permission to update repo in this namespace")
+ }
+ }
+ }
+
+ if req.Private != nil {
+ //TODO: only check for dataset this time, need to check for other types later
+ if repo.RepositoryType == types.DatasetRepo {
+ // check repo sensitivity check status
+ if !*req.Private && repo.SensitiveCheckStatus != types.SensitiveCheckPass {
+ return nil, fmt.Errorf("repo cannot be public before sensitive content check pass, current check status:%s", repo.SensitiveCheckStatus)
+ }
+ }
+ repo.Private = *req.Private
+ }
+ if req.Nickname != nil {
+ repo.Nickname = *req.Nickname
+ }
+ if req.Description != nil {
+ repo.Description = *req.Description
+ }
+
+ gitRepoReq := gitserver.UpdateRepoReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Nickname: repo.Nickname,
+ Description: repo.Description,
+ DefaultBranch: repo.DefaultBranch,
+ Private: repo.Private,
+ RepoType: req.RepoType,
+ }
+ _, err = c.git.UpdateRepo(ctx, gitRepoReq)
+ if err != nil {
+ slog.Error("fail to update repo in git ", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, fmt.Errorf("fail to update repo in git, error: %w", err)
+ }
+
+ resRepo, err := c.repoStore.UpdateRepo(ctx, *repo)
+ if err != nil {
+ slog.Error("fail to update repo in git ", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, fmt.Errorf("fail to update repo in database, error: %w", err)
+ }
+
+ return resRepo, nil
+}
+
+func (c *repoComponentImpl) DeleteRepo(ctx context.Context, req types.DeleteRepoReq) (*database.Repository, error) {
+ repo, err := c.repoStore.Find(ctx, req.Namespace, string(req.RepoType), req.Name)
+ if err != nil {
+ return nil, errors.New("repository does not exist")
+ }
+
+ namespace, err := c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, errors.New("namespace does not exist")
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+
+ if namespace.NamespaceType == database.OrgNamespace {
+ canWrite, err := c.CheckCurrentUserPermission(ctx, req.Username, req.Namespace, membership.RoleAdmin)
+ if err != nil {
+ return nil, err
+ }
+ if !canWrite {
+ return nil, errors.New("users do not have permission to delete repo in this organization")
+ }
+ } else {
+ if namespace.Path != user.Username {
+ return nil, errors.New("users do not have permission to delete repo in this namespace")
+ }
+ }
+
+ err = c.repoStore.CleanRelationsByRepoID(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("fail to clean repo relations, %w", err)
+ }
+
+ deleteRepoReq := gitserver.DeleteRepoReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ RepoType: req.RepoType,
+ }
+ err = c.git.DeleteRepo(ctx, deleteRepoReq)
+ if err != nil {
+ slog.Error("fail to update repo in git ", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, fmt.Errorf("fail to delete repo in git, error: %w", err)
+ }
+
+ err = c.repoStore.DeleteRepo(ctx, *repo)
+ if err != nil {
+ slog.Error("fail to delete repo in git ", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, fmt.Errorf("fail to delete repo in database, error: %w", err)
+ }
+
+ return repo, nil
+}
+
+// PublicToUser gets visible repos of the given user and user's orgs
+func (c *repoComponentImpl) PublicToUser(ctx context.Context, repoType types.RepositoryType, userName string, filter *types.RepoFilter, per, page int) (repos []*database.Repository, count int, err error) {
+ var repoOwnerIDs []int64
+ var isAdmin bool
+
+ if len(userName) > 0 {
+ // get user orgs from user service
+ user, err := c.userSvcClient.GetUserInfo(ctx, userName, userName)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get user info, error: %w", err)
+ }
+
+ dbUser := &database.User{
+ RoleMask: strings.Join(user.Roles, ","),
+ }
+
+ isAdmin = dbUser.CanAdmin()
+
+ if !isAdmin {
+ repoOwnerIDs = append(repoOwnerIDs, user.ID)
+ //get user's orgs
+ for _, org := range user.Orgs {
+ repoOwnerIDs = append(repoOwnerIDs, org.UserID)
+ }
+ }
+ }
+ repos, count, err = c.repoStore.PublicToUser(ctx, repoType, repoOwnerIDs, filter, per, page, isAdmin)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get user public repos, error: %w", err)
+ }
+
+ return repos, count, nil
+}
+
+// relatedRepos gets all repos related to the given repo, and return them by repo type
+func (c *repoComponentImpl) RelatedRepos(ctx context.Context, repoID int64, currentUser string) (map[types.RepositoryType][]*database.Repository, error) {
+ fromRelations, err := c.repoRelationsStore.From(ctx, repoID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get repo relation from, error: %w", err)
+ }
+ var toRepoIDs []int64
+ for _, rel := range fromRelations {
+ toRepoIDs = append(toRepoIDs, rel.ToRepoID)
+ }
+
+ toRelations, err := c.repoRelationsStore.To(ctx, repoID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get repo relation to, error: %w", err)
+ }
+ var fromRepoIDs []int64
+ for _, rel := range toRelations {
+ fromRepoIDs = append(fromRepoIDs, rel.FromRepoID)
+ }
+
+ // combine from and to related repos and then remove duplicates
+ var relatedRepoIDs []int64
+ relatedRepoIDs = append(relatedRepoIDs, toRepoIDs...)
+ relatedRepoIDs = append(relatedRepoIDs, fromRepoIDs...)
+ slices.Sort(relatedRepoIDs)
+ relatedRepoIDs = slices.Compact(relatedRepoIDs)
+
+ var opts []database.SelectOption
+ opts = append(opts, database.Columns("id", "repository_type", "path", "user_id", "private", "name",
+ "nickname", "description", "download_count", "updated_at"))
+
+ relatedRepos, err := c.repoStore.FindByIds(ctx, relatedRepoIDs, opts...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get relation to repositories by ids, error: %w", err)
+ }
+
+ relatedRepos, err = c.VisiableToUser(ctx, relatedRepos, currentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check related repositories visiable to user:%s, %w", currentUser, err)
+ }
+ res := make(map[types.RepositoryType][]*database.Repository)
+ for _, repo := range relatedRepos {
+ res[repo.RepositoryType] = append(res[repo.RepositoryType], repo)
+ }
+ return res, nil
+}
+
+func (c *repoComponentImpl) VisiableToUser(ctx context.Context, repos []*database.Repository, currentUser string) ([]*database.Repository, error) {
+ var res []*database.Repository
+ for _, repo := range repos {
+ if repo.Private {
+ if len(currentUser) == 0 {
+ continue
+ }
+ namespace, _ := repo.NamespaceAndName()
+ canRead, err := c.CheckCurrentUserPermission(ctx, currentUser, namespace, membership.RoleRead)
+ if err != nil {
+ return nil, err
+ }
+ if canRead {
+ res = append(res, repo)
+ }
+ } else {
+ res = append(res, repo)
+ }
+ }
+ return res, nil
+}
+
+func (c *repoComponentImpl) CreateFile(ctx context.Context, req *types.CreateFileReq) (*types.CreateFileResp, error) {
+ slog.Debug("creating file get request", slog.String("namespace", req.Namespace), slog.String("filepath", req.FilePath))
+ var (
+ err error
+ user database.User
+ useLfs bool
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanWrite {
+ return nil, ErrUnauthorized
+ }
+
+ user, err = c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check user, cause: %w", err)
+ }
+ req.Email = user.Email
+
+ _, err = c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check namespace, cause: %w", err)
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ useLfs, req = c.checkIfShouldUseLfs(ctx, req)
+ }
+
+ // TODO:check sensitive content of file
+ fileName := filepath.Base(req.FilePath)
+ if fileName == "README.md" {
+ err = c.createReadmeFile(ctx, req)
+ } else {
+ err = c.createLibraryFile(ctx, req)
+ }
+
+ if err != nil {
+ slog.Error("failed to create repo file", slog.String("file", req.FilePath), slog.Any("error", err), slog.String("namespace", req.Namespace), slog.String("name", req.Name))
+ }
+
+ if useLfs {
+ objectKey := filepath.Join("lfs", req.Pointer.RelativePath())
+ uploadInfo, err := c.s3Client.PutObject(ctx, c.config.S3.Bucket, objectKey, bytes.NewReader(req.OriginalContent), req.Pointer.Size, minio.PutObjectOptions{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to upload to Minio: %w", err)
+ }
+
+ if uploadInfo.Size != req.Pointer.Size {
+ return nil, fmt.Errorf("uploaded file size does not match expected size: %d != %d", uploadInfo.Size, req.Pointer.Size)
+ }
+ _, err = c.lfsMetaObjectStore.UpdateOrCreate(ctx, database.LfsMetaObject{
+ Oid: req.Pointer.Oid,
+ Size: req.Pointer.Size,
+ RepositoryID: repo.ID,
+ Existing: true,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create LFS meta object: %w", err)
+ }
+ }
+
+ err = c.repoStore.SetUpdateTimeByPath(ctx, req.RepoType, req.Namespace, req.Name, time.Now())
+ if err != nil {
+ slog.Error("failed to set repo update time", slog.Any("error", err), slog.String("repo_type", string(req.RepoType)), slog.String("namespace", req.Namespace), slog.String("name", req.Name))
+ }
+
+ var resp types.CreateFileResp
+ return &resp, nil
+}
+
+func (c *repoComponentImpl) createReadmeFile(ctx context.Context, req *types.CreateFileReq) error {
+ var err error
+ contentDecoded, _ := base64.RawStdEncoding.DecodeString(req.Content)
+ _, err = c.tagComponent.UpdateMetaTags(ctx, getTagScopeByRepoType(req.RepoType), req.Namespace, req.Name, string(contentDecoded))
+ if err != nil {
+ return fmt.Errorf("failed to update meta tags, cause: %w", err)
+ }
+
+ err = c.git.CreateRepoFile(req)
+ if err != nil {
+ return fmt.Errorf("failed to create %s file, cause: %w", req.RepoType, err)
+ }
+
+ return err
+}
+
+func (c *repoComponentImpl) createLibraryFile(ctx context.Context, req *types.CreateFileReq) error {
+ var err error
+ err = c.tagComponent.UpdateLibraryTags(ctx, getTagScopeByRepoType(req.RepoType), req.Namespace, req.Name, "", req.FilePath)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to set %s's tags", req.RepoType), slog.String("namespace", req.Namespace),
+ slog.String("name", req.Name), slog.Any("error", err))
+ return fmt.Errorf("failed to set %s's tags, cause: %w", req.RepoType, err)
+ }
+ err = c.git.CreateRepoFile(req)
+ if err != nil {
+ return err
+ }
+
+ return err
+}
+
+func (c *repoComponentImpl) UpdateFile(ctx context.Context, req *types.UpdateFileReq) (*types.UpdateFileResp, error) {
+ slog.Debug("update file get request", slog.String("namespace", req.Namespace), slog.String("filePath", req.FilePath),
+ slog.String("origin_path", req.OriginPath))
+
+ var (
+ err error
+ user database.User
+ useLfs bool
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanWrite {
+ return nil, ErrUnauthorized
+ }
+
+ user, err = c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check user, cause: %w", err)
+ }
+ req.Email = user.Email
+
+ _, err = c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check namespace, cause: %w", err)
+ }
+
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ useLfs, req = c.checkIfShouldUseLfsUpdate(ctx, req)
+ }
+
+ err = c.git.UpdateRepoFile(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update %s file, cause: %w", req.RepoType, err)
+ }
+
+ // TODO:check sensitive content of file
+ fileName := filepath.Base(req.FilePath)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if fileName == "README.md" {
+ slog.Debug("file is readme", slog.String("content", req.Content))
+ err = c.updateReadmeFile(ctx, req)
+ } else {
+ slog.Debug("file is not readme", slog.String("filePath", req.FilePath), slog.String("originPath", req.OriginPath))
+ err = c.updateLibraryFile(ctx, req)
+ }
+ if err != nil {
+ slog.Error("failed to update file", slog.String("file", req.FilePath), slog.Any("error", err), slog.String("namespace", req.Namespace), slog.String("name", req.Name))
+ }
+
+ if useLfs {
+ objectKey := filepath.Join("lfs", req.Pointer.RelativePath())
+ uploadInfo, err := c.s3Client.PutObject(ctx, c.config.S3.Bucket, objectKey, bytes.NewReader(req.OriginalContent), req.Pointer.Size, minio.PutObjectOptions{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to upload to Minio: %w", err)
+ }
+
+ if uploadInfo.Size != req.Pointer.Size {
+ return nil, fmt.Errorf("uploaded file size does not match expected size: %d != %d", uploadInfo.Size, req.Pointer.Size)
+ }
+ _, err = c.lfsMetaObjectStore.UpdateOrCreate(ctx, database.LfsMetaObject{
+ Oid: req.Pointer.Oid,
+ Size: req.Pointer.Size,
+ RepositoryID: repo.ID,
+ Existing: true,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create LFS meta object: %w", err)
+ }
+ }
+
+ err = c.repoStore.SetUpdateTimeByPath(ctx, req.RepoType, req.Namespace, req.Name, time.Now())
+ if err != nil {
+ slog.Error("failed to set repo update time", slog.Any("error", err), slog.String("repo_type", string(req.RepoType)), slog.String("namespace", req.Namespace), slog.String("name", req.Name))
+ }
+
+ resp := new(types.UpdateFileResp)
+ return resp, nil
+}
+
+func (c *repoComponentImpl) DeleteFile(ctx context.Context, req *types.DeleteFileReq) (*types.DeleteFileResp, error) {
+ slog.Debug("delete file get request", slog.String("namespace", req.Namespace), slog.String("filePath", req.FilePath),
+ slog.String("origin_path", req.OriginPath))
+
+ var (
+ err error
+ user database.User
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanWrite {
+ return nil, ErrUnauthorized
+ }
+
+ user, err = c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check user, cause: %w", err)
+ }
+ req.Email = user.Email
+
+ _, err = c.namespaceStore.FindByPath(ctx, req.Namespace)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check namespace, cause: %w", err)
+ }
+
+ err = c.git.DeleteRepoFile(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to delete %s file, cause: %w", req.RepoType, err)
+ }
+
+ // TODO:check sensitive content of file
+ fileName := filepath.Base(req.FilePath)
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if fileName == "README.md" {
+ slog.Debug("file is readme", slog.String("content", req.Content))
+ err = c.deleteReadmeFile(ctx, req)
+ } else {
+ slog.Debug("file is not readme", slog.String("filePath", req.FilePath), slog.String("originPath", req.OriginPath))
+ err = c.deleteLibraryFile(ctx, req)
+ }
+
+ if err != nil {
+ slog.Error("failed to delete file", slog.String("file", req.FilePath), slog.Any("error", err), slog.String("namespace", req.Namespace), slog.String("name", req.Name))
+ }
+
+ err = c.repoStore.SetUpdateTimeByPath(ctx, req.RepoType, req.Namespace, req.Name, time.Now())
+ if err != nil {
+ slog.Error("failed to set repo update time", slog.Any("error", err), slog.String("repo_type", string(req.RepoType)), slog.String("namespace", req.Namespace), slog.String("name", req.Name))
+ }
+
+ resp := new(types.DeleteFileResp)
+ return resp, nil
+}
+
+func (c *repoComponentImpl) updateLibraryFile(ctx context.Context, req *types.UpdateFileReq) error {
+ err := c.changeLibraryFile(ctx, req.FilePath, req.OriginPath, req.Namespace, req.Name, req.RepoType)
+ return err
+}
+
+func (c *repoComponentImpl) deleteLibraryFile(ctx context.Context, req *types.DeleteFileReq) error {
+ err := c.changeLibraryFile(ctx, req.FilePath, req.OriginPath, req.Namespace, req.Name, req.RepoType)
+ return err
+}
+
+func (c *repoComponentImpl) changeLibraryFile(ctx context.Context, filePath, originPath, namespace, name string, repoType types.RepositoryType) error {
+ var err error
+
+ isFileRenamed := filePath != originPath
+ // need to handle tag change only if file renamed
+ if isFileRenamed {
+ err = c.tagComponent.UpdateLibraryTags(ctx, getTagScopeByRepoType(repoType), namespace, name, originPath, filePath)
+ if err != nil {
+ slog.Error(fmt.Sprintf("failed to set %s's tags", repoType), slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ return fmt.Errorf("failed to set %s's tags, cause: %w", repoType, err)
+ }
+ }
+
+ return err
+}
+
+func (c *repoComponentImpl) updateReadmeFile(ctx context.Context, req *types.UpdateFileReq) error {
+ slog.Debug("file is readme", slog.String("content", req.Content))
+ err := c.changeReadmeFile(ctx, req.Content, req.Namespace, req.Name, req.RepoType)
+ if err != nil {
+ return fmt.Errorf("failed to update meta tags for update readme, cause: %w", err)
+ }
+ return err
+}
+
+func (c *repoComponentImpl) deleteReadmeFile(ctx context.Context, req *types.DeleteFileReq) error {
+ err := c.changeReadmeFile(ctx, req.Content, req.Namespace, req.Name, req.RepoType)
+ if err != nil {
+ return fmt.Errorf("failed to update meta tags for delete readme, cause: %w", err)
+ }
+ return err
+}
+
+func (c *repoComponentImpl) changeReadmeFile(ctx context.Context, content, namespace, name string, repoType types.RepositoryType) error {
+ contentDecoded, _ := base64.RawStdEncoding.DecodeString(content)
+ _, err := c.tagComponent.UpdateMetaTags(ctx, getTagScopeByRepoType(repoType), namespace, name, string(contentDecoded))
+ if err != nil {
+ return fmt.Errorf("failed to update meta tags, cause: %w", err)
+ }
+ return err
+}
+
+func (c *repoComponentImpl) Commits(ctx context.Context, req *types.GetCommitsReq) ([]types.Commit, *types.RepoPageOpts, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, nil, ErrUnauthorized
+ }
+
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ getCommitsReq := gitserver.GetRepoCommitsReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Per: req.Per,
+ Page: req.Page,
+ RepoType: req.RepoType,
+ }
+ commits, pageOpt, err := c.git.GetRepoCommits(ctx, getCommitsReq)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to get git %s repository commits, error: %w", req.RepoType, err)
+ }
+ return commits, pageOpt, nil
+}
+
+func (c *repoComponentImpl) LastCommit(ctx context.Context, req *types.GetCommitsReq) (*types.Commit, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrForbidden
+ }
+
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ getLastCommitReq := gitserver.GetRepoLastCommitReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ RepoType: req.RepoType,
+ }
+ commit, err := c.git.GetRepoLastCommit(ctx, getLastCommitReq)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get git %s repository last commit, error: %w", req.RepoType, err)
+ }
+ return commit, nil
+}
+
+func (c *repoComponentImpl) FileRaw(ctx context.Context, req *types.GetFileReq) (string, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil || repo == nil {
+ return "", fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return "", fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return "", ErrUnauthorized
+ }
+
+ if repo.Source != types.LocalSource && strings.ToLower(req.Path) == "readme.md" {
+ _, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return repo.Readme, nil
+ }
+ }
+ }
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ getFileRawReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ raw, err := c.git.GetRepoFileRaw(ctx, getFileRawReq)
+ if err != nil {
+ return "", fmt.Errorf("failed to get git %s repository file raw, error: %w", req.RepoType, err)
+ }
+ return raw, nil
+}
+
+func (c *repoComponentImpl) DownloadFile(ctx context.Context, req *types.GetFileReq, userName string) (io.ReadCloser, int64, string, error) {
+ var (
+ reader io.ReadCloser
+ downloadUrl string
+ size int64
+ )
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, 0, "", fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, 0, "", fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, 0, "", ErrUnauthorized
+ }
+
+ err = c.repoStore.UpdateRepoFileDownloads(ctx, repo, time.Now(), 1)
+ if err != nil {
+ return nil, 0, "", fmt.Errorf("failed to update %s file download count, error: %w", req.RepoType, err)
+ }
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ if req.Lfs {
+ objectKey := path.Join("lfs", req.Path)
+
+ reqParams := make(url.Values)
+ if req.SaveAs != "" {
+ // allow rename when download through content-disposition header
+ reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", req.SaveAs))
+ }
+ signedUrl, err := c.s3Client.PresignedGetObjectGeo(ctx, c.lfsBucket, objectKey, ossFileExpire, reqParams)
+ if err != nil {
+ return nil, 0, downloadUrl, err
+ }
+ return nil, 0, signedUrl.String(), nil
+ } else {
+ getFileReaderReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ reader, size, err = c.git.GetRepoFileReader(ctx, getFileReaderReq)
+ if err != nil {
+ return nil, 0, "", fmt.Errorf("failed to download git %s repository file, error: %w", req.RepoType, err)
+ }
+ return reader, size, downloadUrl, nil
+ }
+}
+
+func (c *repoComponentImpl) Branches(ctx context.Context, req *types.GetBranchesReq) ([]types.Branch, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ getBranchesReq := gitserver.GetBranchesReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Per: req.Per,
+ Page: req.Page,
+ RepoType: req.RepoType,
+ }
+ bs, err := c.git.GetRepoBranches(ctx, getBranchesReq)
+ if err != nil {
+ if repo.Source != types.LocalSource {
+ return []types.Branch{}, nil
+ }
+ return nil, fmt.Errorf("failed to get git %s repository branches, error: %w", req.RepoType, err)
+ }
+ return bs, nil
+}
+
+func (c *repoComponentImpl) Tags(ctx context.Context, req *types.GetTagsReq) ([]database.Tag, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find %s, error: %w", req.RepoType, err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ tags, err := c.repoStore.Tags(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get %s tags, error: %w", req.RepoType, err)
+ }
+ return tags, nil
+}
+
+func (c *repoComponentImpl) UpdateTags(ctx context.Context, namespace, name string, repoType types.RepositoryType, category, currentUser string, tags []string) error {
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, currentUser, repo)
+ if err != nil {
+ return fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanWrite {
+ return ErrUnauthorized
+ }
+
+ tagScope := getTagScopeByRepoType(repoType)
+ err = c.tagComponent.UpdateRepoTagsByCategory(ctx, tagScope, repo.ID, category, tags)
+
+ return err
+}
+
+func (c *repoComponentImpl) Tree(ctx context.Context, req *types.GetFileReq) ([]*types.File, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ if repo == nil {
+ return nil, fmt.Errorf("repo does not exist, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrForbidden
+ }
+
+ if repo.Source != types.LocalSource {
+ _, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ if req.Path == "" {
+ req.Path = "/"
+ }
+ files, err := c.fileStore.FindByParentPath(ctx, repo.ID, req.Path)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return []*types.File{}, nil
+ } else {
+ return nil, err
+ }
+ }
+ var resFiles []*types.File
+ for _, f := range files {
+ resFiles = append(resFiles, &types.File{
+ Name: f.Name,
+ Path: f.Path,
+ Size: f.Size,
+ Commit: types.Commit{
+ Message: f.LastCommitMessage,
+ CommitterDate: f.LastCommitDate,
+ },
+ })
+ }
+ return resFiles, nil
+ }
+ }
+ }
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ tree, err := c.git.GetRepoFileTree(ctx, getRepoFileTree)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get git %s repository file tree, error: %w", req.RepoType, err)
+ }
+ return tree, nil
+}
+
+func (c *repoComponentImpl) UploadFile(ctx context.Context, req *types.CreateFileReq) error {
+ f, err := c.git.GetRepoFileContents(ctx, gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Branch,
+ Path: req.FilePath,
+ RepoType: req.RepoType,
+ })
+ if err != nil {
+ _, err = c.CreateFile(ctx, req)
+ if err != nil {
+ return fmt.Errorf("fail to create file for file uploading, %w", err)
+ }
+ return nil
+ }
+ var updateFileReq types.UpdateFileReq
+
+ updateFileReq.Username = req.Username
+ updateFileReq.Message = req.Message
+ updateFileReq.Branch = req.Branch
+ updateFileReq.Content = req.Content
+ updateFileReq.Namespace = req.Namespace
+ updateFileReq.Name = req.Name
+ updateFileReq.FilePath = req.FilePath
+ // we need file sha, not commit SHA
+ updateFileReq.SHA = f.SHA
+ updateFileReq.RepoType = req.RepoType
+ updateFileReq.OriginalContent = req.OriginalContent
+ updateFileReq.CurrentUser = req.CurrentUser
+
+ _, err = c.UpdateFile(ctx, &updateFileReq)
+
+ return err
+}
+
+func (c *repoComponentImpl) SDKListFiles(ctx context.Context, repoType types.RepositoryType, namespace, name, ref, userName string) (*types.SDKFiles, error) {
+ var sdkFiles []types.SDKFile
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil || repo == nil {
+ return nil, ErrNotFound
+ }
+
+ canRead, err := c.AllowReadAccessRepo(ctx, repo, userName)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !canRead {
+ return nil, ErrUnauthorized
+ }
+
+ if ref == "" {
+ ref = repo.DefaultBranch
+ }
+
+ filePaths, err := getFilePaths(namespace, name, "", repoType, ref, c.git.GetRepoFileTree)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get all %s files, error: %w", repoType, err)
+ }
+
+ for _, filePath := range filePaths {
+ sdkFiles = append(sdkFiles, types.SDKFile{Filename: filePath})
+ }
+ return &types.SDKFiles{
+ ID: fmt.Sprintf("%s/%s", namespace, name),
+ Siblings: sdkFiles,
+ Private: repo.Private,
+ Downloads: repo.DownloadCount,
+ Likes: repo.Likes,
+ Tags: []string{},
+ SHA: repo.DefaultBranch,
+ }, nil
+}
+
+func (c *repoComponentImpl) IsLfs(ctx context.Context, req *types.GetFileReq) (bool, int64, error) {
+ getFileRawReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ content, err := c.git.GetRepoFileRaw(ctx, getFileRawReq)
+ if err != nil {
+ if err.Error() == ErrNotFoundMessage {
+ return false, -1, ErrNotFound
+ }
+ slog.Error("failed to get %s file raw", string(req.RepoType), slog.String("namespace", req.Namespace), slog.String("name", req.Name), slog.String("path", req.Path))
+ return false, -1, err
+ }
+
+ return strings.HasPrefix(content, LFSPrefix), int64(len(content)), nil
+}
+
+func (c *repoComponentImpl) HeadDownloadFile(ctx context.Context, req *types.GetFileReq, userName string) (*types.File, *types.Commit, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ canRead, err := c.AllowReadAccessRepo(ctx, repo, userName)
+ if err != nil {
+ return nil, nil, err
+ }
+ if !canRead {
+ return nil, nil, ErrUnauthorized
+ }
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ file, err := c.git.GetRepoFileContents(ctx, getFileContentReq)
+ if err != nil {
+ slog.Error("err.Error()", slog.Any("err.Error()", err.Error()))
+ if err.Error() == ErrNotFoundMessage || err.Error() == ErrGetContentsOrList {
+ return nil, nil, ErrNotFound
+ }
+ return nil, nil, fmt.Errorf("failed to download git %s repository file, error: %w", req.RepoType, err)
+ }
+
+ lastCommitReq := gitserver.GetRepoLastCommitReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ RepoType: req.RepoType,
+ }
+
+ lastCommit, err := c.git.GetRepoLastCommit(ctx, lastCommitReq)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to get last commit, error: %w", err)
+ }
+ return file, lastCommit, nil
+}
+
+func (c *repoComponentImpl) SDKDownloadFile(ctx context.Context, req *types.GetFileReq, userName string) (io.ReadCloser, int64, string, error) {
+ var downloadUrl string
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, 0, "", fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ canRead, err := c.AllowReadAccessRepo(ctx, repo, userName)
+ if err != nil {
+ return nil, 0, "", err
+ }
+ if !canRead {
+ return nil, 0, "", ErrUnauthorized
+ }
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+ if req.Lfs {
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ file, err := c.git.GetRepoFileContents(ctx, getFileContentReq)
+ if err != nil {
+ return nil, 0, "", err
+ }
+ objectKey := file.LfsRelativePath
+ objectKey = path.Join("lfs", objectKey)
+ reqParams := make(url.Values)
+ if req.SaveAs != "" {
+ // allow rename when download through content-disposition header
+ reqParams.Set("response-content-disposition", fmt.Sprintf("attachment;filename=%s", req.SaveAs))
+ }
+ signedUrl, err := c.s3Client.PresignedGetObjectGeo(ctx, c.lfsBucket, objectKey, ossFileExpire, reqParams)
+ if err != nil {
+ if err.Error() == ErrNotFoundMessage || err.Error() == ErrGetContentsOrList {
+ return nil, 0, downloadUrl, ErrNotFound
+ }
+ return nil, 0, downloadUrl, err
+ }
+ return nil, 0, signedUrl.String(), nil
+
+ } else {
+ getFileReaderReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ }
+ reader, size, err := c.git.GetRepoFileReader(ctx, getFileReaderReq)
+ if err != nil {
+ if err.Error() == ErrNotFoundMessage {
+ return nil, 0, downloadUrl, ErrNotFound
+ }
+ return nil, 0, "", fmt.Errorf("failed to download git %s repository file, error: %w", req.RepoType, err)
+ }
+ return reader, size, downloadUrl, nil
+ }
+}
+
+// UpdateDownloads increase clone download count for repo by given count
+func (c *repoComponentImpl) UpdateDownloads(ctx context.Context, req *types.UpdateDownloadsReq) error {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find %s, error: %w", req.RepoType, err)
+ }
+
+ err = c.repoStore.UpdateRepoCloneDownloads(ctx, repo, req.Date, req.CloneCount)
+ if err != nil {
+ return fmt.Errorf("failed to update %s download count, error: %w", req.RepoType, err)
+ }
+ return err
+}
+
+// IncrDownloads increase the click download count for repo by 1
+func (c *repoComponentImpl) IncrDownloads(ctx context.Context, repoType types.RepositoryType, namespace, name string) error {
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find %s, error: %w", repoType, err)
+ }
+
+ err = c.repoStore.UpdateRepoFileDownloads(ctx, repo, time.Now(), 1)
+ if err != nil {
+ return fmt.Errorf("failed to incr download count, error: %w", err)
+ }
+ return err
+}
+
+func (c *repoComponentImpl) FileInfo(ctx context.Context, req *types.GetFileReq) (*types.File, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ if req.Ref == "" {
+ req.Ref = repo.DefaultBranch
+ }
+
+ if repo.Source != types.LocalSource && strings.ToLower(req.Path) == "readme.md" {
+ _, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ content := base64.StdEncoding.EncodeToString([]byte(repo.Readme))
+ return &types.File{
+ Name: "readme.md",
+ Path: "readme.md",
+ Content: content,
+ }, nil
+ }
+ }
+ }
+
+ getFileContentReq := gitserver.GetRepoInfoByPathReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Ref,
+ Path: req.Path,
+ RepoType: req.RepoType,
+ MaxFileSize: c.adjustMaxFileSize(req.MaxFileSize),
+ }
+ file, err := c.git.GetRepoFileContents(ctx, getFileContentReq)
+ if err != nil {
+ if errors.Is(err, gitserver.ErrFileTooLarge) {
+ // return basic file info, but tell client not to show the file content
+ file.PreviewCode = types.FilePreviewCodeTooLarge
+ return file, nil
+ }
+ return nil, fmt.Errorf("failed to get git model repository file info, error: %w", err)
+ }
+ decodedFileContent, _ := base64.StdEncoding.DecodeString(file.Content)
+ file.PreviewCode = c.getFilePreviewCode(decodedFileContent)
+ return file, nil
+}
+
+func (c *repoComponentImpl) getFilePreviewCode(fileContent []byte) types.FilePreviewCode {
+ // detect the file content type like text/plain, image/jpeg, etc
+ detectedType := http.DetectContentType(fileContent)
+ switch {
+ case strings.HasPrefix(detectedType, "text"):
+ return types.FilePreviewCodeNormal
+ default:
+ return types.FilePreviewCodeNotText
+ }
+}
+
+func (c *repoComponentImpl) adjustMaxFileSize(maxFileSize int64) int64 {
+ // same with aliyun green check large content size
+ const maxModerationContentSize = 100 * 9000
+ if maxFileSize == 0 || maxFileSize > maxModerationContentSize {
+ maxFileSize = maxModerationContentSize
+ }
+ return maxFileSize
+}
+
+func getTagScopeByRepoType(repoType types.RepositoryType) database.TagScope {
+ switch repoType {
+ case types.ModelRepo:
+ return database.ModelTagScope
+ case types.DatasetRepo:
+ return database.DatasetTagScope
+ case types.CodeRepo:
+ return database.CodeTagScope
+ case types.SpaceRepo:
+ return database.SpaceTagScope
+ case types.PromptRepo:
+ return database.PromptTagScope
+ default:
+ panic("convert repo type to tag scope failed, unknown repo type:" + repoType)
+ }
+}
+
+func (c *repoComponentImpl) AllowReadAccessRepo(ctx context.Context, repo *database.Repository, username string) (bool, error) {
+ if !repo.Private {
+ return true, nil
+ }
+
+ if username == "" {
+ return false, ErrUserNotFound
+ }
+
+ namespace, _ := repo.NamespaceAndName()
+ return c.CheckCurrentUserPermission(ctx, username, namespace, membership.RoleRead)
+}
+
+func (c *repoComponentImpl) AllowReadAccess(ctx context.Context, repoType types.RepositoryType, namespace, name, username string) (bool, error) {
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return false, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ return c.AllowReadAccessRepo(ctx, repo, username)
+}
+
+func (c *repoComponentImpl) AllowWriteAccess(ctx context.Context, repoType types.RepositoryType, namespace, name, username string) (bool, error) {
+ _, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return false, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ if username == "" {
+ return false, ErrUserNotFound
+ }
+
+ return c.CheckCurrentUserPermission(ctx, username, namespace, membership.RoleWrite)
+}
+
+func (c *repoComponentImpl) AllowAdminAccess(ctx context.Context, repoType types.RepositoryType, namespace, name, username string) (bool, error) {
+ _, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return false, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ if username == "" {
+ return false, ErrUserNotFound
+ }
+
+ return c.CheckCurrentUserPermission(ctx, username, namespace, membership.RoleAdmin)
+}
+
+func (c *repoComponentImpl) GetUserRepoPermission(ctx context.Context, userName string, repo *database.Repository) (*types.UserRepoPermission, error) {
+ if userName == "" {
+ //anonymous user only has read permission to public repo
+ return &types.UserRepoPermission{CanRead: !repo.Private, CanWrite: false, CanAdmin: false}, nil
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user '%s' when get user repo permission, error: %w", userName, err)
+ }
+ if user.CanAdmin() {
+ return &types.UserRepoPermission{CanRead: true, CanWrite: true, CanAdmin: true}, nil
+ }
+
+ namespace, _ := repo.NamespaceAndName()
+ ns, err := c.namespaceStore.FindByPath(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find namespace '%s' when get user repo permission, error: %w", namespace, err)
+ }
+
+ if ns.NamespaceType == "user" {
+ //owner has full permission
+ if userName == namespace {
+ return &types.UserRepoPermission{
+ CanRead: true,
+ CanWrite: true,
+ CanAdmin: true,
+ }, nil
+ } else {
+ //other user has read permission to pubic repo
+ return &types.UserRepoPermission{
+ CanRead: !repo.Private, CanWrite: false, CanAdmin: false,
+ }, nil
+ }
+ } else {
+ r, err := c.userSvcClient.GetMemberRole(ctx, namespace, userName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user '%s' member role of org '%s' when get user repo permission, error: %w", userName, namespace, err)
+ }
+
+ return &types.UserRepoPermission{
+ CanRead: r.CanRead() || !repo.Private,
+ CanWrite: r.CanWrite(),
+ CanAdmin: r.CanAdmin(),
+ }, nil
+ }
+}
+
+func (c *repoComponentImpl) CheckCurrentUserPermission(ctx context.Context, userName string, namespace string, role membership.Role) (bool, error) {
+ ns, err := c.namespaceStore.FindByPath(ctx, namespace)
+ if err != nil {
+ return false, fmt.Errorf("fail to find namespace '%s', err:%w", namespace, err)
+ }
+
+ u, err := c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return false, fmt.Errorf("fail to find user '%s', err:%w", userName, err)
+ }
+ if u.CanAdmin() {
+ return true, nil
+ }
+
+ if ns.NamespaceType == "user" {
+ return userName == namespace, nil
+ } else {
+ r, err := c.userSvcClient.GetMemberRole(ctx, namespace, userName)
+ if err != nil {
+ return false, err
+ }
+ switch role {
+ case membership.RoleAdmin:
+ return r.CanAdmin(), nil
+ case membership.RoleWrite:
+ return r.CanWrite(), nil
+ case membership.RoleRead:
+ return r.CanRead(), nil
+ default:
+ return false, fmt.Errorf("unknown role %s", role)
+ }
+ }
+}
+
+func (c *repoComponentImpl) GetCommitWithDiff(ctx context.Context, req *types.GetCommitsReq) (*types.CommitResponse, error) {
+ // get commit diff by commit id
+ if req.Ref == "" {
+ return nil, fmt.Errorf("failed to find request commit id")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+
+ permission, err := c.GetUserRepoPermission(ctx, req.CurrentUser, repo)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+ getCommitReq := gitserver.GetRepoLastCommitReq{
+ Namespace: req.Namespace, // user name or org name
+ Name: req.Name, // repo name
+ Ref: req.Ref, // commit id
+ RepoType: req.RepoType, // repo type
+ }
+ resp, errGSC := c.git.GetSingleCommit(ctx, getCommitReq)
+ if errGSC != nil {
+ return nil, fmt.Errorf("failed to get git %s repository %s commit id '%s', error: %w", req.RepoType, req.Name, req.Ref, errGSC)
+ }
+
+ return resp, nil
+}
+
+func (c *repoComponentImpl) CreateMirror(ctx context.Context, req types.CreateMirrorReq) (*database.Mirror, error) {
+ var (
+ mirror database.Mirror
+ taskId int64
+ )
+ admin, err := c.CheckCurrentUserPermission(ctx, req.CurrentUser, req.Namespace, membership.RoleAdmin)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check permission to create mirror, error: %w", err)
+ }
+
+ if !admin {
+ return nil, fmt.Errorf("users do not have permission to create mirror for this repo")
+ }
+
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ exists, err := c.mirrorStore.IsExist(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ if exists {
+ return nil, fmt.Errorf("mirror already exists")
+ }
+ mirrorSource, err := c.mirrorSourceStore.Get(ctx, req.MirrorSourceID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get mirror source, err: %w, id: %d", err, req.MirrorSourceID)
+ }
+ pushAccessToken, err := c.tokenStore.GetUserGitToken(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find access token, error: %w", err)
+ }
+ mirror.Interval = req.Interval
+ mirror.SourceUrl = req.SourceUrl
+ mirror.MirrorSourceID = req.MirrorSourceID
+ mirror.Username = req.Username
+ mirror.PushUrl = repo.HTTPCloneURL
+ mirror.AccessToken = req.AccessToken
+ mirror.SourceRepoPath = req.SourceRepoPath
+ mirror.LocalRepoPath = fmt.Sprintf("%s_%s_%s_%s", mirrorSource.SourceName, req.RepoType, req.Namespace, req.Name)
+ mirror.RepositoryID = repo.ID
+
+ if c.config.Saas {
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ mirror.PushUsername = req.CurrentUser
+ mirror.PushAccessToken = pushAccessToken.Token
+ taskId, err = c.mirrorServer.CreateMirrorRepo(ctx, mirrorserver.CreateMirrorRepoReq{
+ Namespace: "root",
+ Name: mirror.LocalRepoPath,
+ CloneUrl: mirror.SourceUrl,
+ Username: mirror.Username,
+ AccessToken: mirror.AccessToken,
+ Private: false,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create pull mirror in mirror server: %v", err)
+ }
+ }
+ } else {
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ err = c.git.MirrorSync(ctx, gitserver.MirrorSyncReq{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ CloneUrl: mirror.SourceUrl,
+ Username: mirror.Username,
+ AccessToken: mirror.AccessToken,
+ RepoType: req.RepoType,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to create pull mirror in mirror server: %v", err)
+ }
+ }
+ }
+
+ mirror.MirrorTaskID = taskId
+
+ reqMirror, err := c.mirrorStore.Create(ctx, &mirror)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create mirror")
+ }
+
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ c.mq.PushRepoMirror(&queue.MirrorTask{
+ MirrorID: reqMirror.ID,
+ Priority: queue.PriorityMap[reqMirror.Priority],
+ })
+ reqMirror.Status = types.MirrorWaiting
+ err = c.mirrorStore.Update(ctx, reqMirror)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update mirror status: %v", err)
+ }
+ }
+
+ return reqMirror, nil
+}
+
+func (c *repoComponentImpl) MirrorFromSaas(ctx context.Context, namespace, name, currentUser string, repoType types.RepositoryType) error {
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ m, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ if !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ }
+ if m != nil {
+ err := c.mirrorFromSaasSync(ctx, m, repo, namespace, name, repoType)
+ if err != nil {
+ return fmt.Errorf("failed to trigger mirror sync, error: %w", err)
+ }
+ return nil
+ }
+
+ var mirror database.Mirror
+ syncVersion, err := c.syncVersionStore.FindByRepoTypeAndPath(ctx, repo.PathWithOutPrefix(), repoType)
+ if err != nil {
+ return fmt.Errorf("failed to find sync version, error: %w", err)
+ }
+ mirrorSource := &database.MirrorSource{}
+ if syncVersion.SourceID == types.SyncVersionSourceOpenCSG {
+ mirrorSource.SourceName = types.OpenCSGPrefix
+ } else if syncVersion.SourceID == types.SyncVersionSourceHF {
+ mirrorSource.SourceName = types.HuggingfacePrefix
+ }
+
+ mirrorSource.SourceName = types.OpenCSGPrefix
+ syncClientSetting, err := c.syncClientSettingStore.First(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to find sync client setting, error: %w", err)
+ }
+
+ sourceUrl := common.TrimPrefixCloneURLBySourceID(c.config.MultiSync.SaasSyncDomain, string(repoType), namespace, name, syncVersion.SourceID)
+ mirror.SourceUrl = sourceUrl
+ mirror.MirrorSourceID = mirrorSource.ID
+ mirror.RepositoryID = repo.ID
+ mirror.Username = currentUser
+ mirror.AccessToken = c.config.Mirror.Token
+ mirror.SourceRepoPath = fmt.Sprintf("%s/%s", namespace, name)
+
+ taskId, err := c.git.CreateMirrorRepo(ctx, gitserver.CreateMirrorRepoReq{
+ Namespace: namespace,
+ Name: name,
+ CloneUrl: mirror.SourceUrl,
+ RepoType: repoType,
+ MirrorToken: syncClientSetting.Token,
+ Private: false,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to create pull mirror in mirror server: %v", err)
+ }
+
+ mirror.MirrorTaskID = taskId
+
+ _, err = c.mirrorStore.Create(ctx, &mirror)
+
+ if err != nil {
+ return fmt.Errorf("failed to create mirror")
+ }
+
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ c.mq.PushRepoMirror(&queue.MirrorTask{
+ MirrorID: mirror.ID,
+ Priority: queue.Priority(mirror.Priority),
+ CreatedAt: mirror.CreatedAt.Unix(),
+ MirrorToken: syncClientSetting.Token,
+ })
+ repo.SyncStatus = types.SyncStatusPending
+ } else {
+ repo.SyncStatus = types.SyncStatusInProgress
+ }
+ _, err = c.repoStore.UpdateRepo(ctx, *repo)
+ if err != nil {
+ return fmt.Errorf("failed to update repo sync status: %w", err)
+ }
+ return nil
+}
+
+func (c *repoComponentImpl) mirrorFromSaasSync(ctx context.Context, mirror *database.Mirror, repo *database.Repository, namespace, name string, repoType types.RepositoryType) error {
+ var err error
+ syncClientSetting, err := c.syncClientSettingStore.First(ctx)
+ if err != nil {
+ return fmt.Errorf("failed to find sync client setting, error: %w", err)
+ }
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ err = c.git.MirrorSync(ctx, gitserver.MirrorSyncReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: repoType,
+ MirrorToken: syncClientSetting.Token,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to sync mirror, error: %w", err)
+ }
+ repo.SyncStatus = types.SyncStatusInProgress
+ }
+
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ c.mq.PushRepoMirror(&queue.MirrorTask{
+ MirrorID: mirror.ID,
+ Priority: queue.Priority(mirror.Priority),
+ CreatedAt: mirror.CreatedAt.Unix(),
+ MirrorToken: syncClientSetting.Token,
+ })
+ repo.SyncStatus = types.SyncStatusPending
+ }
+
+ _, err = c.repoStore.UpdateRepo(ctx, *repo)
+ if err != nil {
+ return fmt.Errorf("failed to update repo sync status: %w", err)
+ }
+
+ return nil
+}
+
+func (c *repoComponentImpl) GetMirror(ctx context.Context, req types.GetMirrorReq) (*database.Mirror, error) {
+ admin, err := c.CheckCurrentUserPermission(ctx, req.CurrentUser, req.Namespace, membership.RoleAdmin)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check permission to create mirror, error: %w", err)
+ }
+
+ if !admin {
+ return nil, fmt.Errorf("users do not have permission to get mirror for this repo")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ mirror, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ return mirror, nil
+}
+
+func (c *repoComponentImpl) UpdateMirror(ctx context.Context, req types.UpdateMirrorReq) (*database.Mirror, error) {
+ admin, err := c.CheckCurrentUserPermission(ctx, req.CurrentUser, req.Namespace, membership.RoleAdmin)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check permission to create mirror, error: %w", err)
+ }
+
+ if !admin {
+ return nil, fmt.Errorf("users do not have permission to update mirror for this repo")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ mirror, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ mirrorSource, err := c.mirrorSourceStore.Get(ctx, req.MirrorSourceID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get mirror source, err: %w, id: %d", err, req.MirrorSourceID)
+ }
+
+ pushAccessToken, err := c.tokenStore.GetUserGitToken(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find access token, error: %w", err)
+ }
+
+ mirror.Interval = req.Interval
+ mirror.SourceUrl = req.SourceUrl
+ mirror.MirrorSourceID = req.MirrorSourceID
+ mirror.Username = req.Username
+ mirror.PushUrl = req.PushUrl
+ mirror.AccessToken = req.AccessToken
+ mirror.PushUsername = req.CurrentUser
+ mirror.PushAccessToken = pushAccessToken.Token
+ mirror.SourceRepoPath = req.SourceRepoPath
+ mirror.LocalRepoPath = fmt.Sprintf("%s_%s_%s_%s", mirrorSource.SourceName, req.RepoType, req.Namespace, req.Name)
+ err = c.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update mirror, error: %w", err)
+ }
+ return mirror, nil
+}
+
+func (c *repoComponentImpl) DeleteMirror(ctx context.Context, req types.DeleteMirrorReq) error {
+ admin, err := c.CheckCurrentUserPermission(ctx, req.CurrentUser, req.Namespace, membership.RoleAdmin)
+ if err != nil {
+ return fmt.Errorf("failed to check permission to create mirror, error: %w", err)
+ }
+
+ if !admin {
+ return fmt.Errorf("users do not have permission to delete mirror for this repo")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ mirror, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ return fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ err = c.mirrorStore.Delete(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to delete mirror, error: %w", err)
+ }
+ return nil
+}
+
+// get runtime framework list with type
+func (c *repoComponentImpl) ListRuntimeFrameworkWithType(ctx context.Context, deployType int) ([]types.RuntimeFramework, error) {
+ frames, err := c.runtimeFrameworksStore.List(ctx, deployType)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list runtime frameworks, error: %w", err)
+ }
+ var frameList []types.RuntimeFramework
+ for _, frame := range frames {
+ frameList = append(frameList, types.RuntimeFramework{
+ ID: frame.ID,
+ FrameName: frame.FrameName,
+ FrameVersion: frame.FrameVersion,
+ FrameImage: frame.FrameImage,
+ FrameCpuImage: frame.FrameCpuImage,
+ Enabled: frame.Enabled,
+ ContainerPort: frame.ContainerPort,
+ Type: frame.Type,
+ })
+ }
+ return frameList, nil
+}
+
+// get runtime framework list
+func (c *repoComponentImpl) ListRuntimeFramework(ctx context.Context, repoType types.RepositoryType, namespace, name string, deployType int) ([]types.RuntimeFramework, error) {
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ frames, err := c.runtimeFrameworksStore.ListByRepoID(ctx, repo.ID, deployType)
+ if err != nil {
+ return nil, fmt.Errorf("failed to list runtime frameworks, error: %w", err)
+ }
+ var frameList []types.RuntimeFramework
+ for _, modelFrame := range frames {
+ if modelFrame.RuntimeFramework != nil {
+ frameList = append(frameList, types.RuntimeFramework{
+ ID: modelFrame.RuntimeFramework.ID,
+ FrameName: modelFrame.RuntimeFramework.FrameName,
+ FrameVersion: modelFrame.RuntimeFramework.FrameVersion,
+ FrameImage: modelFrame.RuntimeFramework.FrameImage,
+ FrameCpuImage: modelFrame.RuntimeFramework.FrameCpuImage,
+ FrameNpuImage: modelFrame.RuntimeFramework.FrameNpuImage,
+ Enabled: modelFrame.RuntimeFramework.Enabled,
+ ContainerPort: modelFrame.RuntimeFramework.ContainerPort,
+ })
+ }
+ }
+ return frameList, nil
+}
+
+func (c *repoComponentImpl) CreateRuntimeFramework(ctx context.Context, req *types.RuntimeFrameworkReq) (*types.RuntimeFramework, error) {
+ newFrame := database.RuntimeFramework{
+ FrameName: req.FrameName,
+ FrameVersion: req.FrameVersion,
+ FrameImage: req.FrameImage,
+ FrameCpuImage: req.FrameCpuImage,
+ Enabled: req.Enabled,
+ ContainerPort: req.ContainerPort,
+ Type: req.Type,
+ }
+ err := c.runtimeFrameworksStore.Add(ctx, newFrame)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create runtime framework, error: %w", err)
+ }
+ frame := &types.RuntimeFramework{
+ FrameName: req.FrameName,
+ FrameVersion: req.FrameVersion,
+ FrameImage: req.FrameImage,
+ FrameCpuImage: req.FrameCpuImage,
+ Enabled: req.Enabled,
+ ContainerPort: req.ContainerPort,
+ Type: req.Type,
+ }
+ return frame, nil
+}
+
+func (c *repoComponentImpl) UpdateRuntimeFramework(ctx context.Context, id int64, req *types.RuntimeFrameworkReq) (*types.RuntimeFramework, error) {
+ newFrame := database.RuntimeFramework{
+ ID: id,
+ FrameName: req.FrameName,
+ FrameVersion: req.FrameVersion,
+ FrameImage: req.FrameImage,
+ FrameCpuImage: req.FrameCpuImage,
+ Enabled: req.Enabled,
+ ContainerPort: req.ContainerPort,
+ Type: req.Type,
+ }
+ frame, err := c.runtimeFrameworksStore.Update(ctx, newFrame)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update runtime frameworks, error: %w", err)
+ }
+ return &types.RuntimeFramework{
+ ID: frame.ID,
+ FrameName: frame.FrameName,
+ FrameVersion: frame.FrameVersion,
+ FrameImage: frame.FrameImage,
+ FrameCpuImage: frame.FrameCpuImage,
+ Enabled: frame.Enabled,
+ ContainerPort: frame.ContainerPort,
+ Type: req.Type,
+ }, nil
+}
+
+func (c *repoComponentImpl) DeleteRuntimeFramework(ctx context.Context, id int64) error {
+ frame, err := c.runtimeFrameworksStore.FindByID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to find runtime frameworks, error: %w", err)
+ }
+ err = c.runtimeFrameworksStore.Delete(ctx, *frame)
+ return err
+}
+
+func (c *repoComponentImpl) ListDeploy(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string) ([]types.DeployRepo, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ slog.Error("Failed to query deploy", slog.Any("error", err), slog.Any("repotype", repoType), slog.Any("namespace", namespace), slog.Any("name", name))
+ return nil, errors.New("invalid repository for query parameters")
+ }
+ if repo == nil {
+ slog.Error("nothing found for deploys", slog.Any("repotype", repoType), slog.Any("namespace", namespace), slog.Any("name", name))
+ return nil, errors.New("nothing found for deploys")
+ }
+ deploys, err := c.deployTaskStore.ListDeploy(ctx, repoType, repo.ID, user.ID)
+ if err != nil {
+ return nil, errors.New("fail to list user deploys")
+ }
+ var resDeploys []types.DeployRepo
+ for _, deploy := range deploys {
+ resDeploys = append(resDeploys, types.DeployRepo{
+ DeployID: deploy.ID,
+ DeployName: deploy.DeployName,
+ RepoID: deploy.RepoID,
+ SvcName: deploy.SvcName,
+ Status: deployStatusCodeToString(deploy.Status),
+ Hardware: deploy.Hardware,
+ Env: deploy.Env,
+ RuntimeFramework: deploy.RuntimeFramework,
+ ImageID: deploy.ImageID,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ GitBranch: deploy.GitBranch,
+ ClusterID: deploy.ClusterID,
+ SecureLevel: deploy.SecureLevel,
+ CreatedAt: deploy.CreatedAt,
+ UpdatedAt: deploy.UpdatedAt,
+ })
+ }
+ return resDeploys, nil
+}
+
+func (c *repoComponentImpl) DeleteDeploy(ctx context.Context, delReq types.DeployActReq) error {
+ user, deploy, err := c.checkDeployPermissionForUser(ctx, delReq)
+ if err != nil {
+ return err
+ }
+ // delete service
+ deployRepo := types.DeployRepo{
+ SpaceID: 0,
+ DeployID: delReq.DeployID,
+ Namespace: delReq.Namespace,
+ Name: delReq.Name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ }
+ // purge service
+ err = c.deployer.Purge(ctx, deployRepo)
+ if err != nil {
+ // fail to purge deploy instance, maybe service is gone
+ slog.Warn("purge deploy instance", slog.Any("error", err))
+ }
+
+ exist, err := c.deployer.Exist(ctx, deployRepo)
+ if err != nil {
+ // fail to check service
+ return err
+ }
+
+ if exist {
+ // fail to delete service
+ return errors.New("fail to delete service")
+ }
+
+ // update database deploy
+ err = c.deployTaskStore.DeleteDeploy(ctx, types.RepositoryType(delReq.RepoType), deploy.RepoID, user.ID, delReq.DeployID)
+ if err != nil {
+ return fmt.Errorf("fail to remove deploy instance, %w", err)
+ }
+ // release resource if it's a order case
+ if deploy.OrderDetailID != 0 {
+ ur, err := c.userResourcesStore.FindUserResourcesByOrderDetailId(ctx, deploy.UserUUID, deploy.OrderDetailID)
+ if err != nil {
+ return fmt.Errorf("fail to find user resource, %w", err)
+ }
+ ur.DeployId = 0
+ err = c.userResourcesStore.UpdateDeployId(ctx, ur)
+ if err != nil {
+ return fmt.Errorf("fail to release resource, %w", err)
+ }
+
+ }
+
+ return err
+}
+
+func (c *repoComponentImpl) DeployDetail(ctx context.Context, detailReq types.DeployActReq) (*types.DeployRepo, error) {
+ var (
+ deploy *database.Deploy
+ err error
+ )
+ if detailReq.DeployType == types.ServerlessType {
+ _, deploy, err = c.checkDeployPermissionForServerless(ctx, detailReq)
+ } else {
+ _, deploy, err = c.checkDeployPermissionForUser(ctx, detailReq)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ endpoint, _ := c.GenerateEndpoint(ctx, deploy)
+
+ req := types.DeployRepo{
+ DeployID: deploy.ID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: detailReq.Namespace,
+ Name: detailReq.Name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ }
+ actualReplica, desiredReplica, instList, err := c.deployer.GetReplica(ctx, req)
+ if err != nil {
+ slog.Warn("fail to get deploy replica", slog.Any("repotype", detailReq.RepoType), slog.Any("req", req), slog.Any("error", err))
+ }
+ endpointPrivate := true
+ if deploy.SecureLevel == types.EndpointPublic {
+ endpointPrivate = false
+ }
+ proxyEndPoint := ""
+ if deploy.Type == types.FinetuneType {
+ proxyEndPoint = endpoint + "/proxy/7860/"
+ }
+ repoPath := strings.TrimPrefix(deploy.GitPath, string(detailReq.RepoType)+"s_")
+ resDeploy := types.DeployRepo{
+ DeployID: deploy.ID,
+ DeployName: deploy.DeployName,
+ RepoID: deploy.RepoID,
+ SvcName: deploy.SvcName,
+ Status: deployStatusCodeToString(deploy.Status),
+ Hardware: deploy.Hardware,
+ Env: deploy.Env,
+ RuntimeFramework: deploy.RuntimeFramework,
+ ImageID: deploy.ImageID,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ GitBranch: deploy.GitBranch,
+ ClusterID: deploy.ClusterID,
+ SecureLevel: deploy.SecureLevel,
+ CreatedAt: deploy.CreatedAt,
+ UpdatedAt: deploy.UpdatedAt,
+ Endpoint: endpoint,
+ ActualReplica: actualReplica,
+ DesiredReplica: desiredReplica,
+ Instances: instList,
+ Private: endpointPrivate,
+ Path: repoPath,
+ ProxyEndpoint: proxyEndPoint,
+ SKU: deploy.SKU,
+ }
+
+ return &resDeploy, nil
+}
+
+// generate endpoint
+func (c *repoComponentImpl) GenerateEndpoint(ctx context.Context, deploy *database.Deploy) (string, string) {
+ var endpoint string
+ provider := ""
+ cls, err := c.clusterInfoStore.ByClusterID(ctx, deploy.ClusterID)
+ zone := ""
+ if err != nil {
+ slog.Warn("Get cluster with error", slog.Any("error", err))
+ } else {
+ zone = cls.Zone
+ provider = cls.Provider
+ }
+ if len(deploy.SvcName) > 0 && deploy.Status == deployStatus.Running {
+ // todo: zone.provider.endpoint to support multi-zone, multi-provider
+ regionDomain := ""
+ if len(zone) > 0 && len(provider) > 0 {
+ regionDomain = fmt.Sprintf(".%s.%s", zone, provider)
+ }
+ if c.publicRootDomain == "" {
+ endpoint, _ = url.JoinPath(c.serverBaseUrl, "endpoint", deploy.SvcName)
+ endpoint = strings.Replace(endpoint, "http://", "", 1)
+ endpoint = strings.Replace(endpoint, "https://", "", 1)
+ } else {
+ endpoint = fmt.Sprintf("%s%s.%s", deploy.SvcName, regionDomain, c.publicRootDomain)
+ }
+
+ }
+
+ return endpoint, provider
+}
+
+func deployStatusCodeToString(code int) string {
+ // DeployBuildPending = 10
+ // DeployBuildInProgress = 11
+ // DeployBuildFailed = 12
+ // DeployBuildSucceed = 13
+ // DeployBuildSkip = 14
+ //
+ // DeployPrepareToRun = 20
+ // DeployStartUp = 21
+ // DeployRunning = 22
+ // DeployRunTimeError = 23
+ // DeployStopped = 26
+ // DeployRunDeleted = 27 // end user trigger delete action for deploy
+
+ // simplified status for frontend show
+ var txt string
+ switch code {
+ case 10:
+ txt = SpaceStatusStopped
+ case 11:
+ txt = SpaceStatusBuilding
+ case 12:
+ txt = SpaceStatusBuildFailed
+ case 13:
+ txt = SpaceStatusDeploying
+ case 20:
+ txt = SpaceStatusDeploying
+ case 21:
+ txt = SpaceStatusDeployFailed
+ case 22:
+ txt = SpaceStatusDeploying
+ case 23:
+ txt = SpaceStatusRunning
+ case 24:
+ txt = SpaceStatusRuntimeError
+ case 25:
+ txt = SpaceStatusSleeping
+ case 26:
+ txt = SpaceStatusStopped
+ case 27:
+ txt = RepoStatusDeleted
+ default:
+ txt = SpaceStatusStopped
+ }
+ return txt
+}
+
+func (c *repoComponentImpl) DeployInstanceLogs(ctx context.Context, logReq types.DeployActReq) (*deploy.MultiLogReader, error) {
+ var (
+ deploy *database.Deploy
+ err error
+ )
+ if logReq.DeployType == types.ServerlessType {
+ _, deploy, err = c.checkDeployPermissionForServerless(ctx, logReq)
+ } else {
+ _, deploy, err = c.checkDeployPermissionForUser(ctx, logReq)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ return c.deployer.InstanceLogs(ctx, types.DeployRepo{
+ DeployID: deploy.ID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: logReq.Namespace,
+ Name: logReq.Name,
+ ClusterID: deploy.ClusterID,
+ SvcName: deploy.SvcName,
+ InstanceName: logReq.InstanceName,
+ })
+}
+
+// check access repo permission by repo id
+func (c *repoComponentImpl) AllowAccessByRepoID(ctx context.Context, repoID int64, username string) (bool, error) {
+ r, err := c.repoStore.FindById(ctx, repoID)
+ if err != nil {
+ return false, fmt.Errorf("failed to get repository by repo_id: %d, %w", repoID, err)
+ }
+ if r == nil {
+ return false, fmt.Errorf("invalid repository by repo_id: %d", repoID)
+ }
+ fields := strings.Split(r.Path, "/")
+ return c.AllowReadAccess(ctx, r.RepositoryType, fields[0], fields[1], username)
+}
+
+// check access endpoint for rproxy
+func (c *repoComponentImpl) AllowAccessEndpoint(ctx context.Context, currentUser string, deploy *database.Deploy) (bool, error) {
+ if deploy.SecureLevel == types.EndpointPublic {
+ // public endpoint
+ return true, nil
+ }
+ return c.checkAccessDeployForUser(ctx, deploy.RepoID, currentUser, deploy)
+}
+
+// check access deploy permission
+func (c *repoComponentImpl) AllowAccessDeploy(ctx context.Context, req types.DeployActReq) (bool, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return false, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ if repo == nil {
+ return false, fmt.Errorf("failed to find %s repo %s/%s", req.RepoType, req.Namespace, req.Name)
+ }
+ deploy, err := c.deployTaskStore.GetDeployByID(ctx, req.DeployID)
+ if err != nil {
+ return false, err
+ }
+ if deploy == nil {
+ return false, fmt.Errorf("fail to get deploy by ID: %v", req.DeployID)
+ }
+ if req.DeployType == types.ServerlessType {
+ return c.checkAccessDeployForServerless(ctx, repo.ID, req.CurrentUser, deploy)
+ } else {
+ return c.checkAccessDeployForUser(ctx, repo.ID, req.CurrentUser, deploy)
+ }
+}
+
+// common check function for apiserver and rproxy
+func (c *repoComponentImpl) checkAccessDeployForUser(ctx context.Context, repoID int64, currentUser string, deploy *database.Deploy) (bool, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return false, errors.New("user does not exist")
+ }
+ if deploy.UserID != user.ID {
+ // deny access due to deploy was not created by
+ return false, &types.PermissionError{Message: "deploy was not created by user"}
+ }
+ if deploy.RepoID != repoID {
+ // deny access for invalid repo
+ return false, errors.New("invalid deploy found")
+ }
+ return true, nil
+}
+
+func (c *repoComponentImpl) checkAccessDeployForServerless(ctx context.Context, repoID int64, currentUser string, deploy *database.Deploy) (bool, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return false, fmt.Errorf("user %s does not exist", currentUser)
+ }
+ isAdmin := c.IsAdminRole(user)
+ if !isAdmin {
+ return false, errors.New("need admin permission to see Serverless deploy instances")
+ }
+ if deploy.RepoID != repoID {
+ // deny access for invalid repo
+ return false, errors.New("invalid deploy found")
+ }
+ return true, nil
+}
+
+func (c *repoComponentImpl) DeployStop(ctx context.Context, stopReq types.DeployActReq) error {
+ var (
+ user *database.User
+ deploy *database.Deploy
+ err error
+ )
+ if stopReq.DeployType == types.ServerlessType {
+ user, deploy, err = c.checkDeployPermissionForServerless(ctx, stopReq)
+ } else {
+ user, deploy, err = c.checkDeployPermissionForUser(ctx, stopReq)
+ }
+ if err != nil {
+ return fmt.Errorf("fail to check permission for stop deploy, %w", err)
+ }
+ // delete service
+ deployRepo := types.DeployRepo{
+ DeployID: stopReq.DeployID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: stopReq.Namespace,
+ Name: stopReq.Name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ OrderDetailID: deploy.OrderDetailID,
+ UserUUID: user.UUID,
+ }
+ err = c.deployer.Stop(ctx, deployRepo)
+ if err != nil {
+ // fail to stop deploy instance, maybe service is gone
+ slog.Warn("stop deploy instance with error", slog.Any("error", err), slog.Any("stopReq", stopReq))
+ }
+
+ exist, err := c.deployer.Exist(ctx, deployRepo)
+ if err != nil {
+ // fail to check service
+ return err
+ }
+
+ if exist {
+ // fail to delete service
+ return errors.New("fail to stop deploy instance")
+ }
+
+ // update database deploy to stopped
+ err = c.deployTaskStore.StopDeploy(ctx, stopReq.RepoType, deploy.RepoID, user.ID, stopReq.DeployID)
+ if err != nil {
+ return fmt.Errorf("fail to stop deploy instance, %w", err)
+ }
+
+ return err
+}
+
+func (c *repoComponentImpl) AllowReadAccessByDeployID(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string, deployID int64) (bool, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return false, errors.New("user does not exist")
+ }
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return false, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ deploy, err := c.deployTaskStore.GetDeployByID(ctx, deployID)
+ if err != nil {
+ return false, err
+ }
+ if deploy == nil {
+ return false, errors.New("fail to get deploy by ID")
+ }
+ if deploy.UserID != user.ID {
+ return false, errors.New("deploy was not created by user")
+ }
+ if deploy.RepoID != repo.ID {
+ return false, errors.New("found incorrect repo")
+ }
+ return c.AllowReadAccessRepo(ctx, repo, currentUser)
+}
+
+func (c *repoComponentImpl) DeployStatus(ctx context.Context, repoType types.RepositoryType, namespace, name string, deployID int64) (string, string, []types.Instance, error) {
+ deploy, err := c.deployTaskStore.GetDeployByID(ctx, deployID)
+ if err != nil {
+ return "", SpaceStatusStopped, nil, err
+ }
+ // request deploy status by deploy id
+ srvName, code, instances, err := c.deployer.Status(ctx, types.DeployRepo{
+ DeployID: deploy.ID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: namespace,
+ Name: name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ }, true)
+ if err != nil {
+ slog.Error("error happen when get deploy status", slog.Any("error", err), slog.String("path", deploy.GitPath))
+ return "", SpaceStatusStopped, instances, err
+ }
+ return srvName, deployStatusCodeToString(code), instances, nil
+}
+
+func (c *repoComponentImpl) GetDeployBySvcName(ctx context.Context, svcName string) (*database.Deploy, error) {
+ d, err := c.deployTaskStore.GetDeployBySvcName(ctx, svcName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get deploy by svc name:%s, %w", svcName, err)
+ }
+ if d == nil {
+ return nil, fmt.Errorf("do not found deploy by svc name:%s", svcName)
+ }
+ return d, nil
+}
+
+func (c *repoComponentImpl) SyncMirror(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string) error {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ admin, err := c.CheckCurrentUserPermission(ctx, currentUser, namespace, membership.RoleAdmin)
+ if err != nil {
+ return fmt.Errorf("failed to check permission to create mirror, error: %w", err)
+ }
+
+ if !admin {
+ return fmt.Errorf("users do not have permission to sync mirror for this repo")
+ }
+ }
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ mirror, err := c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ return fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ mirror.Priority = types.HighMirrorPriority
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ err = c.mirrorServer.MirrorSync(ctx, mirrorserver.MirrorSyncReq{
+ Namespace: "root",
+ Name: mirror.LocalRepoPath,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to sync mirror, error: %w", err)
+ }
+ } else if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ c.mq.PushRepoMirror(&queue.MirrorTask{
+ MirrorID: mirror.ID,
+ Priority: queue.PriorityMap[mirror.Priority],
+ CreatedAt: mirror.CreatedAt.Unix(),
+ })
+ mirror.Status = types.MirrorWaiting
+ err = c.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror status: %v", err)
+ }
+ }
+ return nil
+}
+
+func (c *repoComponentImpl) MirrorProgress(ctx context.Context, repoType types.RepositoryType, namespace, name, currentUser string) ([]types.LFSSyncProgressResp, error) {
+ var progressResp []types.LFSSyncProgressResp
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, errors.New("user does not exist")
+ }
+ if !user.CanAdmin() {
+ admin, err := c.CheckCurrentUserPermission(ctx, currentUser, namespace, membership.RoleAdmin)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check permission to create mirror, error: %w", err)
+ }
+
+ if !admin {
+ return nil, fmt.Errorf("users do not have permission to check mirror progress for this repo")
+ }
+ }
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ _, err = c.mirrorStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find mirror, error: %w", err)
+ }
+ lfsMetaObjects, err := c.lfsMetaObjectStore.FindByRepoID(ctx, repo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find lfs meta objects by repo id, error: %w", err)
+ }
+ repoPath := fmt.Sprintf("%ss/%s", repo.RepositoryType, repo.Path)
+ for _, lfsMetaObject := range lfsMetaObjects {
+ progress, _ := c.syncCache.GetLfsSyncFileProgress(ctx, repoPath, lfsMetaObject.Oid)
+ progressResp = append(progressResp, types.LFSSyncProgressResp{
+ Oid: lfsMetaObject.Oid,
+ Progress: progress,
+ })
+ }
+
+ return progressResp, nil
+}
+
+func (c *repoComponentImpl) checkDeployPermissionForUser(ctx context.Context, deployReq types.DeployActReq) (*database.User, *database.Deploy, error) {
+ user, err := c.userStore.FindByUsername(ctx, deployReq.CurrentUser)
+ if err != nil {
+ return nil, nil, &types.PermissionError{Message: "user does not exist"}
+ }
+ deploy, err := c.deployTaskStore.GetDeployByID(ctx, deployReq.DeployID)
+ if err != nil {
+ return nil, nil, fmt.Errorf("fail to get user deploy %v, %w", deployReq.DeployID, err)
+ }
+ if deploy == nil {
+ return nil, nil, fmt.Errorf("do not found user deploy %v", deployReq.DeployID)
+ }
+ if deploy.UserID != user.ID {
+ return nil, nil, &types.PermissionError{Message: "deploy was not created by user"}
+ }
+ return &user, deploy, nil
+}
+
+func (c *repoComponentImpl) checkDeployPermissionForServerless(ctx context.Context, deployReq types.DeployActReq) (*database.User, *database.Deploy, error) {
+ user, err := c.userStore.FindByUsername(ctx, deployReq.CurrentUser)
+ if err != nil {
+ return nil, nil, fmt.Errorf("user does not exist, %w", err)
+ }
+ isAdmin := c.IsAdminRole(user)
+ if !isAdmin {
+ return nil, nil, fmt.Errorf("need admin permission for Serverless deploy")
+ }
+ deploy, err := c.deployTaskStore.GetDeployByID(ctx, deployReq.DeployID)
+ if err != nil {
+ return nil, nil, fmt.Errorf("fail to get serverless deploy:%v, %w", deployReq.DeployID, err)
+ }
+ if deploy == nil {
+ return nil, nil, fmt.Errorf("do not found serverless deploy %v", deployReq.DeployID)
+ }
+ return &user, deploy, nil
+}
+
+func (c *repoComponentImpl) DeployUpdate(ctx context.Context, updateReq types.DeployActReq, req *types.DeployUpdateReq) error {
+ var (
+ deploy *database.Deploy
+ err error
+ )
+ if updateReq.DeployType == types.ServerlessType {
+ _, deploy, err = c.checkDeployPermissionForServerless(ctx, updateReq)
+ } else {
+ _, deploy, err = c.checkDeployPermissionForUser(ctx, updateReq)
+ }
+ if err != nil {
+ return fmt.Errorf("fail to check permission for update deploy, %w", err)
+ }
+ // check user balance if resource changed
+ if req.ResourceID != nil {
+ // don't support switch reserved resource
+ if deploy.OrderDetailID != 0 {
+ return fmt.Errorf("don't support switch reserved resource so far")
+ }
+ resource, err := c.spaceResourceStore.FindByID(ctx, *req.ResourceID)
+ if err != nil {
+ return fmt.Errorf("fail to find resource by id, %w", err)
+ }
+
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)),
+ ResourceID: strconv.FormatInt(int64(resource.ID), 10),
+ })
+ if err != nil {
+ return fmt.Errorf("fail to find price data, %w", err)
+ }
+ if priceData.Total == 0 {
+ return fmt.Errorf("cannot find valid price data")
+ }
+ // check user balance
+ if priceData.Prices[0].SkuPrice > 0 {
+ account, err := c.accountingComponent.QueryBalanceByUserIDInternal(ctx, updateReq.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("cannot find user balance, %w", err)
+ }
+ if account.Balance <= 0 {
+ return fmt.Errorf("balance is not enough to start resources. current balance: %.2f", account.Balance/100)
+ }
+ }
+ frame, err := c.runtimeFrameworksStore.FindEnabledByName(ctx, deploy.RuntimeFramework)
+ if err != nil {
+ return fmt.Errorf("cannot find available runtime framework, %w", err)
+ }
+ //update runtime image once user changed cpu to gpu
+ req.RuntimeFrameworkID = &frame.ID
+ }
+
+ if req.ClusterID != nil {
+ _, err = c.clusterInfoStore.ByClusterID(ctx, *req.ClusterID)
+ if err != nil {
+ return fmt.Errorf("invalid cluster %v, %w", *req.ClusterID, err)
+ }
+ }
+
+ // check service
+ deployRepo := types.DeployRepo{
+ DeployID: updateReq.DeployID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: updateReq.Namespace,
+ Name: updateReq.Name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ }
+ exist, err := c.deployer.Exist(ctx, deployRepo)
+ if err != nil {
+ return fmt.Errorf("check deploy exists, err: %w", err)
+ }
+
+ if exist {
+ // deploy instance is running
+ return errors.New("stop deploy first")
+ }
+
+ // update inference service and keep deploy_id and svc_name unchanged
+ err = c.deployer.UpdateDeploy(ctx, req, deploy)
+ return err
+}
+
+func (c *repoComponentImpl) DeployStart(ctx context.Context, startReq types.DeployActReq) error {
+ var (
+ deploy *database.Deploy
+ err error
+ )
+ if startReq.DeployType == types.ServerlessType {
+ _, deploy, err = c.checkDeployPermissionForServerless(ctx, startReq)
+ } else {
+ _, deploy, err = c.checkDeployPermissionForUser(ctx, startReq)
+ }
+
+ if err != nil {
+ return fmt.Errorf("failed to check permission for start deploy, %w", err)
+ }
+ // check user balance
+ resourceId, err := strconv.ParseInt(deploy.SKU, 10, 64)
+ if err != nil {
+ return fmt.Errorf("failed to parse resource id, %w", err)
+ }
+ resource, err := c.spaceResourceStore.FindByID(ctx, resourceId)
+ if err != nil {
+ return fmt.Errorf("failed to find resource, %w", err)
+ }
+ // check user balance
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ ResourceID: strconv.FormatInt(int64(resource.ID), 10),
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)),
+ })
+ if err != nil {
+ return fmt.Errorf("fail to find price data, %w", err)
+ }
+ if priceData.Total == 0 {
+ return fmt.Errorf("cannot find valid price data")
+ }
+
+ if priceData.Prices[0].SkuPrice > 0 && deploy.OrderDetailID == 0 {
+ account, err := c.accountingComponent.QueryBalanceByUserIDInternal(ctx, startReq.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("cannot find user balance, %w", err)
+ }
+ if account.Balance <= 0 {
+ return fmt.Errorf("balance is not enough to start resources. current balance: %.2f", account.Balance/100)
+ }
+ }
+ // check service
+ deployRepo := types.DeployRepo{
+ DeployID: startReq.DeployID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: startReq.Namespace,
+ Name: startReq.Name,
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ }
+ exist, err := c.deployer.Exist(ctx, deployRepo)
+ if err != nil {
+ return err
+ }
+
+ if exist {
+ // deploy instance is running
+ return errors.New("stop deploy first")
+ }
+
+ // start deploy
+ err = c.deployer.StartDeploy(ctx, deploy)
+ if err != nil {
+ return fmt.Errorf("fail to start deploy, %w", err)
+ }
+
+ return err
+}
+
+func (c *repoComponentImpl) AllFiles(ctx context.Context, req types.GetAllFilesReq) ([]*types.File, error) {
+ repo, err := c.repoStore.FindByPath(ctx, req.RepoType, req.Namespace, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ if repo == nil {
+ return nil, fmt.Errorf("failed to find repo")
+ }
+ if repo.Private {
+ read, err := c.CheckCurrentUserPermission(ctx, req.CurrentUser, req.Namespace, membership.RoleRead)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check permission to get all files, error: %w", err)
+ }
+
+ if !read {
+ return nil, fmt.Errorf("users do not have permission to get all files for this repo")
+ }
+ }
+ allFiles, err := getAllFiles(req.Namespace, req.Name, "", req.RepoType, req.Ref, c.git.GetRepoFileTree)
+ if err != nil {
+ slog.Error("fail to get all files of repository", slog.Any("repoType", req.RepoType), slog.String("namespace", req.Namespace), slog.String("name", req.Name), slog.String("error", err.Error()))
+ return nil, err
+ }
+ return allFiles, nil
+}
+
+func (c *repoComponentImpl) IsAdminRole(user database.User) bool {
+ slog.Debug("Check if user is admin", slog.Any("user", user))
+ return user.CanAdmin()
+}
+
+func (c *repoComponentImpl) GetNameSpaceInfo(ctx context.Context, path string) (*types.Namespace, error) {
+ nsResp, err := c.userSvcClient.GetNameSpaceInfo(ctx, path)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace infor from user service, path: %s, error: %w", path, err)
+ }
+ ns := &types.Namespace{
+ Path: nsResp.Path,
+ Avatar: nsResp.Avatar,
+ Type: nsResp.Type,
+ }
+ return ns, nil
+}
+
+func (c *repoComponentImpl) checkIfShouldUseLfs(ctx context.Context, req *types.CreateFileReq) (bool, *types.CreateFileReq) {
+ gFile, err := c.git.GetRepoFileContents(ctx, gitserver.GetRepoInfoByPathReq{
+ RepoType: req.RepoType,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Branch,
+ Path: GitAttributesFileName,
+ })
+ if err != nil {
+ return false, req
+ }
+ fileName := filepath.Base(req.FilePath)
+ decodedBytes, _ := base64.StdEncoding.DecodeString(gFile.Content)
+ attributes := parseGitattributesContent(string(decodedBytes))
+ useLfs := shouldUseLFS(fileName, attributes)
+ if !useLfs {
+ return false, req
+ }
+ pointer, pointerFile := generateLFSPointerFromContent([]byte(req.OriginalContent))
+ req.Content = pointerFile
+ req.Pointer = pointer
+ return true, req
+}
+
+func (c *repoComponentImpl) checkIfShouldUseLfsUpdate(ctx context.Context, req *types.UpdateFileReq) (bool, *types.UpdateFileReq) {
+ gFile, err := c.git.GetRepoFileContents(ctx, gitserver.GetRepoInfoByPathReq{
+ RepoType: req.RepoType,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ Ref: req.Branch,
+ Path: GitAttributesFileName,
+ })
+ if err != nil {
+ return false, req
+ }
+ fileName := filepath.Base(req.FilePath)
+ decodedBytes, _ := base64.StdEncoding.DecodeString(gFile.Content)
+ attributes := parseGitattributesContent(string(decodedBytes))
+ useLfs := shouldUseLFS(fileName, attributes)
+ if !useLfs {
+ return false, req
+ }
+ pointer, pointerFile := generateLFSPointerFromContent([]byte(req.OriginalContent))
+ req.Content = pointerFile
+ req.Pointer = pointer
+ return true, req
+}
+
+func parseGitattributesContent(content string) map[string][]string {
+ attributes := make(map[string][]string)
+ scanner := bufio.NewScanner(strings.NewReader(content))
+
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+ parts := strings.Fields(line)
+ if len(parts) >= 2 {
+ attributes[parts[0]] = parts[1:]
+ }
+ }
+
+ return attributes
+}
+
+func shouldUseLFS(filePath string, attributes map[string][]string) bool {
+ filePath = strings.ToLower(filePath)
+ for pattern, attrs := range attributes {
+ matched, _ := filepath.Match(pattern, filePath)
+ if matched {
+ for _, attr := range attrs {
+ if attr == "filter=lfs" || attr == "lfs" {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+func generateLFSPointerFromContent(content []byte) (*types.Pointer, string) {
+ hash := sha256.New()
+
+ hash.Write(content)
+ oid := fmt.Sprintf("sha256:%x", hash.Sum(nil))
+
+ fileSize := int64(len(content))
+
+ lfsPointerContent := fmt.Sprintf(`version https://git-lfs.github.com/spec/v1
+oid %s
+size %d
+`, oid, fileSize)
+ encodingContent := base64.StdEncoding.EncodeToString([]byte(lfsPointerContent))
+ pointer := types.Pointer{
+ Oid: strings.Split(oid, ":")[1],
+ Size: fileSize,
+ }
+
+ return &pointer, encodingContent
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "sync"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type repoFileComponentImpl struct {
+ rfs database.RepoFileStore
+ rs database.RepoStore
+ gs gitserver.GitServer
+}
+
+type RepoFileComponent interface {
+ GenRepoFileRecords(ctx context.Context, repoType types.RepositoryType, namespace, name string) error
+ GenRepoFileRecordsBatch(ctx context.Context, repoType types.RepositoryType, lastRepoID int64, concurrency int) error
+}
+
+func NewRepoFileComponent(conf *config.Config) (RepoFileComponent, error) {
+ c := &repoFileComponentImpl{
+ rfs: database.NewRepoFileStore(),
+ rs: database.NewRepoStore(),
+ }
+ gs, err := git.NewGitServer(conf)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server, error: %w", err)
+ }
+
+ c.gs = gs
+ return c, nil
+}
+func (c *repoFileComponentImpl) GenRepoFileRecords(ctx context.Context, repoType types.RepositoryType, namespace, name string) error {
+ repo, err := c.rs.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ return c.createRepoFileRecords(ctx, *repo, "", c.gs.GetRepoFileTree)
+}
+
+func (c *repoFileComponentImpl) GenRepoFileRecordsBatch(ctx context.Context, repoType types.RepositoryType, lastRepoID int64, concurrency int) error {
+ tokens := make(chan struct{}, concurrency)
+ for i := 0; i < concurrency; i++ {
+ tokens <- struct{}{}
+ }
+ wg := &sync.WaitGroup{}
+ //TODO: load last repo id from redis cache
+ batch := 10
+ for {
+ repos, err := c.rs.BatchGet(ctx, repoType, lastRepoID, batch)
+ if err != nil {
+ return fmt.Errorf("failed to get repos in batch, error: %w", err)
+ }
+ for _, repo := range repos {
+ //wait
+ <-tokens
+ wg.Add(1)
+ go func(repo database.Repository) {
+ slog.Info("start to get files of repository", slog.Any("repoType", repoType), slog.String("path", repo.Path))
+ //get file paths of repo
+ err := c.createRepoFileRecords(ctx, repo, "", c.gs.GetRepoFileTree)
+ if err != nil {
+ slog.Error("fail to get all files of repository",
+ slog.String("path", repo.Path), slog.String("repo_type", string(repo.RepositoryType)),
+ slog.String("error", err.Error()))
+ }
+ tokens <- struct{}{}
+ wg.Done()
+ }(repo)
+
+ }
+
+ if len(repos) < batch {
+ break
+ }
+ lastRepoID = repos[len(repos)-1].ID
+ }
+
+ wg.Wait()
+ return nil
+}
+
+func (c *repoFileComponentImpl) createRepoFileRecords(ctx context.Context, repo database.Repository, folder string, gsTree func(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error)) error {
+ namespace, name := repo.NamespaceAndName()
+ var files []*types.File
+
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: "",
+ Path: folder,
+ RepoType: repo.RepositoryType,
+ }
+ gitFiles, err := gsTree(context.Background(), getRepoFileTree)
+ if err != nil {
+ return fmt.Errorf("failed to get repo file tree,%w", err)
+ }
+ for _, file := range gitFiles {
+ if file.Type == "dir" {
+ err := c.createRepoFileRecords(ctx, repo, file.Path, gsTree)
+ if err != nil {
+ return err
+ }
+ } else {
+ files = append(files, file)
+ }
+ }
+ //get all files
+ for _, file := range files {
+ // save repo files into db
+ rf := database.RepositoryFile{
+ RepositoryID: repo.ID,
+ Path: file.Path,
+ FileType: file.Type,
+ Size: file.Size,
+ CommitSha: file.SHA,
+ LfsRelativePath: file.LfsRelativePath,
+ Branch: repo.DefaultBranch,
+ }
+
+ var exists bool
+ var err error
+ if exists, err = c.rfs.Exists(ctx, rf); err != nil {
+ slog.Error("failed to check repository file exists", slog.Any("repo_id", repo.ID),
+ slog.String("file_path", rf.Path), slog.String("error", err.Error()))
+ continue
+ }
+
+ if exists {
+ slog.Info("skip create exist repository file", slog.Any("repo_id", repo.ID), slog.String("file_path", rf.Path))
+ continue
+ }
+ if err := c.rfs.Create(ctx, &rf); err != nil {
+ slog.Error("failed to save repository file", slog.Any("repo_id", repo.ID),
+ slog.String("error", err.Error()))
+ return fmt.Errorf("failed to save repository file, error: %w", err)
+ }
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "strings"
+ "sync"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var (
+ MainBranch string = "main"
+ MasterBranch string = "master"
+ ConfigFileName string = "config.json"
+ ScanLock sync.Mutex
+)
+
+type runtimeArchitectureComponentImpl struct {
+ repoComponent RepoComponent
+ repoStore database.RepoStore
+ repoRuntimeFrameworkStore database.RepositoriesRuntimeFrameworkStore
+ runtimeArchStore database.RuntimeArchitecturesStore
+ runtimeFrameworksStore database.RuntimeFrameworksStore
+ tagStore database.TagStore
+ resouceModelStore database.ResourceModelStore
+ gitServer gitserver.GitServer
+}
+
+type RuntimeArchitectureComponent interface {
+ ListByRuntimeFrameworkID(ctx context.Context, id int64) ([]database.RuntimeArchitecture, error)
+ SetArchitectures(ctx context.Context, id int64, architectures []string) ([]string, error)
+ DeleteArchitectures(ctx context.Context, id int64, architectures []string) ([]string, error)
+ ScanArchitecture(ctx context.Context, id int64, scanType int, models []string) error
+ // check if it's supported model resource by name
+ IsSupportedModelResource(ctx context.Context, modelName string, rfm *database.RuntimeFramework, id int64) (bool, error)
+ GetArchitectureFromConfig(ctx context.Context, namespace, name string) (string, error)
+ // remove runtime_framework tag from model
+ RemoveRuntimeFrameworkTag(ctx context.Context, rftags []*database.Tag, repoId, rfId int64)
+ // add runtime_framework tag to model
+ AddRuntimeFrameworkTag(ctx context.Context, rftags []*database.Tag, repoId, rfId int64) error
+ // add resource tag to model
+ AddResourceTag(ctx context.Context, rstags []*database.Tag, modelname string, repoId int64) error
+}
+
+func NewRuntimeArchitectureComponent(config *config.Config) (RuntimeArchitectureComponent, error) {
+ c := &runtimeArchitectureComponentImpl{}
+ c.runtimeFrameworksStore = database.NewRuntimeFrameworksStore()
+ c.runtimeArchStore = database.NewRuntimeArchitecturesStore()
+ c.tagStore = database.NewTagStore()
+ c.resouceModelStore = database.NewResourceModelStore()
+ repo, err := NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create repo component, %w", err)
+ }
+ c.repoComponent = repo
+ c.repoStore = database.NewRepoStore()
+ c.repoRuntimeFrameworkStore = database.NewRepositoriesRuntimeFramework()
+ c.gitServer, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ return c, nil
+}
+
+func (c *runtimeArchitectureComponentImpl) ListByRuntimeFrameworkID(ctx context.Context, id int64) ([]database.RuntimeArchitecture, error) {
+ archs, err := c.runtimeArchStore.ListByRuntimeFrameworkID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("list runtime arch failed, %w", err)
+ }
+ return archs, nil
+}
+
+func (c *runtimeArchitectureComponentImpl) SetArchitectures(ctx context.Context, id int64, architectures []string) ([]string, error) {
+ _, err := c.runtimeFrameworksStore.FindByID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("invalid runtime framework id, %w", err)
+ }
+ var failedArchs []string
+ for _, arch := range architectures {
+ if len(strings.Trim(arch, " ")) < 1 {
+ continue
+ }
+ err := c.runtimeArchStore.Add(ctx, database.RuntimeArchitecture{
+ RuntimeFrameworkID: id,
+ ArchitectureName: strings.Trim(arch, " "),
+ })
+ if err != nil {
+ failedArchs = append(failedArchs, arch)
+ }
+ }
+ return failedArchs, nil
+}
+
+func (c *runtimeArchitectureComponentImpl) DeleteArchitectures(ctx context.Context, id int64, architectures []string) ([]string, error) {
+ _, err := c.runtimeFrameworksStore.FindByID(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("invalid runtime framework id, %w", err)
+ }
+ var failedDeletes []string
+ for _, arch := range architectures {
+ if len(strings.Trim(arch, " ")) < 1 {
+ continue
+ }
+ err := c.runtimeArchStore.DeleteByRuntimeIDAndArchName(ctx, id, strings.Trim(arch, " "))
+ if err != nil {
+ failedDeletes = append(failedDeletes, arch)
+ }
+ }
+ return failedDeletes, nil
+}
+
+func (c *runtimeArchitectureComponentImpl) ScanArchitecture(ctx context.Context, id int64, scanType int, models []string) error {
+ frame, err := c.runtimeFrameworksStore.FindByID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("invalid runtime framework id, %w", err)
+ }
+ archs, err := c.runtimeArchStore.ListByRuntimeFrameworkID(ctx, id)
+ if err != nil {
+ return fmt.Errorf("list runtime arch failed, %w", err)
+ }
+ var archMap map[string]string = make(map[string]string)
+ for _, arch := range archs {
+ archMap[arch.ArchitectureName] = arch.ArchitectureName
+ }
+
+ if ScanLock.TryLock() {
+ go func() {
+ slog.Info("scan models to update runtime frameworks started")
+ defer ScanLock.Unlock()
+ if scanType == 0 || scanType == 2 {
+ err := c.scanExistModels(ctx, types.ScanReq{
+ FrameID: id,
+ FrameType: frame.Type,
+ ArchMap: archMap,
+ Models: models,
+ })
+ if err != nil {
+ slog.Any("scan old models failed", slog.Any("error", err))
+ }
+ }
+
+ if scanType == 0 || scanType == 1 {
+ err := c.scanNewModels(ctx, types.ScanReq{
+ FrameID: id,
+ FrameType: frame.Type,
+ ArchMap: archMap,
+ Models: models,
+ })
+ if err != nil {
+ slog.Any("scan new models failed", slog.Any("error", err))
+ }
+ }
+ slog.Info("scan models to update runtime frameworks done")
+ }()
+ } else {
+ return fmt.Errorf("architecture scan is already in progress")
+ }
+ return nil
+}
+
+func (c *runtimeArchitectureComponentImpl) scanNewModels(ctx context.Context, req types.ScanReq) error {
+ repos, err := c.repoStore.GetRepoWithoutRuntimeByID(ctx, req.FrameID, req.Models)
+ if err != nil {
+ return fmt.Errorf("failed to get repos without runtime by ID, %w", err)
+ }
+ if repos == nil {
+ return nil
+ }
+ runtime_framework, err := c.runtimeFrameworksStore.FindByID(ctx, req.FrameID)
+ if err != nil {
+ return fmt.Errorf("failed to get runtime framework by ID, %w", err)
+ }
+ runtime_framework_tags, _ := c.tagStore.GetTagsByScopeAndCategories(ctx, "model", []string{"runtime_framework", "resource"})
+ for _, repo := range repos {
+ namespace, name := repo.NamespaceAndName()
+ arch, err := c.GetArchitectureFromConfig(ctx, namespace, name)
+ if err != nil {
+ slog.Warn("did not to get arch for create relation", slog.Any("ConfigFileName", ConfigFileName), slog.Any("repo", repo.Path), slog.Any("error", err))
+ continue
+ }
+ if len(arch) < 1 {
+ continue
+ }
+ // check if model is in resource model table but not in runtime framework repo
+ isSupportedRM, err := c.IsSupportedModelResource(ctx, name, runtime_framework, repo.ID)
+ if err != nil {
+ slog.Debug("fail to check model name in runtime framework repo", slog.Any("repo", repo.Path), slog.Any("error", err))
+ }
+ _, exist := req.ArchMap[arch]
+ if !exist && !isSupportedRM {
+ continue
+ }
+ err = c.repoRuntimeFrameworkStore.Add(ctx, req.FrameID, repo.ID, req.FrameType)
+ if err != nil {
+ slog.Warn("fail to create relation", slog.Any("repo", repo.Path), slog.Any("frameid", req.FrameID), slog.Any("error", err))
+ }
+ // add runtime framework and resource tag to model
+ err = c.AddRuntimeFrameworkTag(ctx, runtime_framework_tags, repo.ID, req.FrameID)
+ if err != nil {
+ slog.Warn("fail to add runtime framework tag", slog.Any("repo", repo.Path), slog.Any("frameid", req.FrameID), slog.Any("error", err))
+ }
+ err = c.AddResourceTag(ctx, runtime_framework_tags, name, repo.ID)
+ if err != nil {
+ slog.Warn("fail to add resource tag", slog.Any("repo", repo.Path), slog.Any("frameid", req.FrameID), slog.Any("error", err))
+ }
+ }
+ return nil
+}
+
+// check if it's supported model resource by name
+func (c *runtimeArchitectureComponentImpl) IsSupportedModelResource(ctx context.Context, modelName string, rf *database.RuntimeFramework, id int64) (bool, error) {
+ trimModel := strings.Replace(strings.ToLower(modelName), "meta-", "", 1)
+ rm, err := c.resouceModelStore.CheckModelNameNotInRFRepo(ctx, trimModel, id)
+ if err != nil || rm == nil {
+ return false, err
+ }
+ image := strings.ToLower(rf.FrameImage)
+ if strings.Contains(image, "/") {
+ parts := strings.Split(image, "/")
+ image = parts[len(parts)-1]
+ }
+
+ if strings.Contains(image, rm.EngineName) {
+ return true, nil
+ }
+ if strings.Contains(rf.FrameNpuImage, rm.EngineName) {
+ return true, nil
+ }
+ // special handling for nim models
+ nimImage := strings.ReplaceAll(image, "-", "")
+ nimMatchModel := strings.ReplaceAll(trimModel, "-", "")
+ if strings.Contains(nimImage, nimMatchModel) {
+ return true, nil
+ }
+ return false, nil
+}
+
+func (c *runtimeArchitectureComponentImpl) scanExistModels(ctx context.Context, req types.ScanReq) error {
+ repos, err := c.repoStore.GetRepoWithRuntimeByID(ctx, req.FrameID, req.Models)
+ if err != nil {
+ return fmt.Errorf("fail to get repos with runtime by ID, %w", err)
+ }
+ if repos == nil {
+ return nil
+ }
+ for _, repo := range repos {
+ fields := strings.Split(repo.Path, "/")
+ arch, err := c.GetArchitectureFromConfig(ctx, fields[0], fields[1])
+ if err != nil {
+ slog.Warn("did not to get arch for remove relation", slog.Any("ConfigFileName", ConfigFileName), slog.Any("repo", repo.Path), slog.Any("error", err))
+ continue
+ }
+ if len(arch) < 1 {
+ continue
+ }
+ _, exist := req.ArchMap[arch]
+ if exist {
+ continue
+ }
+ err = c.repoRuntimeFrameworkStore.Delete(ctx, req.FrameID, repo.ID, req.FrameType)
+ if err != nil {
+ slog.Warn("fail to remove relation", slog.Any("repo", repo.Path), slog.Any("frameid", req.FrameID), slog.Any("error", err))
+ }
+ }
+ return nil
+}
+
+func (c *runtimeArchitectureComponentImpl) GetArchitectureFromConfig(ctx context.Context, namespace, name string) (string, error) {
+ content, err := c.getConfigContent(ctx, namespace, name)
+ if err != nil {
+ return "", fmt.Errorf("fail to read config.json for relation, %w", err)
+ }
+ var config struct {
+ Architectures []string `json:"architectures"`
+ }
+ if err := json.Unmarshal([]byte(content), &config); err != nil {
+ return "", fmt.Errorf("fail to unmarshal config, %w", err)
+ }
+ slog.Debug("unmarshal config", slog.Any("config", config))
+ if config.Architectures == nil {
+ return "", nil
+ }
+ if len(config.Architectures) < 1 {
+ return "", nil
+ }
+ slog.Debug("architectures of config", slog.Any("Architectures", config.Architectures))
+ return config.Architectures[0], nil
+}
+
+func (c *runtimeArchitectureComponentImpl) getConfigContent(ctx context.Context, namespace, name string) (string, error) {
+ content, err := c.gitServer.GetRepoFileRaw(ctx, gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: MainBranch,
+ Path: ConfigFileName,
+ RepoType: types.ModelRepo,
+ })
+ if err != nil {
+ return "", fmt.Errorf("get RepoFileRaw for relation, %w", err)
+ }
+ return content, nil
+}
+
+// remove runtime_framework tag from model
+func (c *runtimeArchitectureComponentImpl) RemoveRuntimeFrameworkTag(ctx context.Context, rftags []*database.Tag, repoId, rfId int64) {
+ rfw, _ := c.runtimeFrameworksStore.FindByID(ctx, rfId)
+ for _, tag := range rftags {
+ if strings.Contains(rfw.FrameImage, tag.Name) || strings.Contains(rfw.FrameNpuImage, tag.Name) {
+ err := c.tagStore.RemoveRepoTags(ctx, repoId, []int64{tag.ID})
+ if err != nil {
+ slog.Warn("fail to remove runtime_framework tag from model repo", slog.Any("repoId", repoId), slog.Any("runtime_framework_id", rfId), slog.Any("error", err))
+ }
+ }
+ }
+}
+
+// add runtime_framework tag to model
+func (c *runtimeArchitectureComponentImpl) AddRuntimeFrameworkTag(ctx context.Context, rftags []*database.Tag, repoId, rfId int64) error {
+ rfw, err := c.runtimeFrameworksStore.FindByID(ctx, rfId)
+ if err != nil {
+ return err
+ }
+ for _, tag := range rftags {
+ if strings.Contains(rfw.FrameImage, tag.Name) || strings.Contains(rfw.FrameNpuImage, tag.Name) {
+ err := c.tagStore.UpsertRepoTags(ctx, repoId, []int64{}, []int64{tag.ID})
+ if err != nil {
+ slog.Warn("fail to add runtime_framework tag to model repo", slog.Any("repoId", repoId), slog.Any("runtime_framework_id", rfId), slog.Any("error", err))
+ }
+ }
+ }
+ return nil
+}
+
+// add resource tag to model
+func (c *runtimeArchitectureComponentImpl) AddResourceTag(ctx context.Context, rstags []*database.Tag, modelname string, repoId int64) error {
+ rms, err := c.resouceModelStore.FindByModelName(ctx, modelname)
+ if err != nil {
+ return err
+ }
+ for _, rm := range rms {
+ for _, tag := range rstags {
+ if strings.Contains(rm.ResourceName, tag.Name) {
+ err := c.tagStore.UpsertRepoTags(ctx, repoId, []int64{}, []int64{tag.ID})
+ if err != nil {
+ slog.Warn("fail to add resource tag to model repo", slog.Any("repoId", repoId), slog.Any("error", err))
+ }
+ }
+ }
+
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type sensitiveComponentImpl struct {
+ checker rpc.ModerationSvcClient
+}
+
+type SensitiveComponent interface {
+ CheckText(ctx context.Context, scenario, text string) (bool, error)
+ CheckImage(ctx context.Context, scenario, ossBucketName, ossObjectName string) (bool, error)
+ CheckRequestV2(ctx context.Context, req types.SensitiveRequestV2) (bool, error)
+}
+
+func NewSensitiveComponent(cfg *config.Config) (SensitiveComponent, error) {
+ if !cfg.SensitiveCheck.Enable {
+ return &sensitiveComponentNoOpImpl{}, nil
+ }
+
+ c := &sensitiveComponentImpl{}
+ c.checker = rpc.NewModerationSvcHttpClient(fmt.Sprintf("%s:%d", cfg.Moderation.Host, cfg.Moderation.Port))
+ return c, nil
+}
+
+func (c sensitiveComponentImpl) CheckText(ctx context.Context, scenario, text string) (bool, error) {
+ result, err := c.checker.PassTextCheck(ctx, scenario, text)
+ if err != nil {
+ return false, err
+ }
+
+ return !result.IsSensitive, nil
+}
+
+func (c sensitiveComponentImpl) CheckImage(ctx context.Context, scenario, ossBucketName, ossObjectName string) (bool, error) {
+ result, err := c.checker.PassImageCheck(ctx, scenario, ossBucketName, ossObjectName)
+ if err != nil {
+ return false, err
+ }
+ return !result.IsSensitive, nil
+}
+
+func (c sensitiveComponentImpl) CheckRequestV2(ctx context.Context, req types.SensitiveRequestV2) (bool, error) {
+ fields := req.GetSensitiveFields()
+ for _, field := range fields {
+ if len(field.Value()) == 0 {
+ continue
+ }
+ result, err := c.checker.PassTextCheck(ctx, field.Scenario, field.Value())
+ if err != nil {
+ slog.Error("fail to check request sensitivity", slog.String("field", field.Name), slog.Any("error", err))
+ return false, fmt.Errorf("fail to check '%s' sensitivity, error: %w", field.Name, err)
+ }
+ if result.IsSensitive {
+ slog.Error("found sensitive words in request", slog.String("field", field.Name))
+ return false, errors.New("found sensitive words in field: " + field.Name)
+ }
+ }
+ return true, nil
+}
+
+// sensitiveComponentNoOpImpl this implementation provides a "no-op" (no operation) version of the SensitiveComponent interface,
+// where all methods simply return a "not sensitive" result without performing any actual checks.
+type sensitiveComponentNoOpImpl struct {
+}
+
+func (c *sensitiveComponentNoOpImpl) CheckText(ctx context.Context, scenario, text string) (bool, error) {
+ return true, nil
+}
+
+func (c *sensitiveComponentNoOpImpl) CheckImage(ctx context.Context, scenario, ossBucketName, ossObjectName string) (bool, error) {
+ return true, nil
+}
+
+// implements SensitiveComponent
+func (c *sensitiveComponentNoOpImpl) CheckRequestV2(ctx context.Context, req types.SensitiveRequestV2) (bool, error) {
+ return true, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/deploy/scheduler"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+const spaceGitattributesContent = modelGitattributesContent
+
+var (
+ streamlitConfigContent = `[server]
+enableCORS = false
+enableXsrfProtection = false
+`
+ streamlitConfig = ".streamlit/config.toml"
+)
+
+type SpaceComponent interface {
+ Create(ctx context.Context, req types.CreateSpaceReq) (*types.Space, error)
+ Show(ctx context.Context, namespace, name, currentUser string) (*types.Space, error)
+ Update(ctx context.Context, req *types.UpdateSpaceReq) (*types.Space, error)
+ Index(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Space, int, error)
+ OrgSpaces(ctx context.Context, req *types.OrgSpacesReq) ([]types.Space, int, error)
+ // UserSpaces get spaces of owner and visible to current user
+ UserSpaces(ctx context.Context, req *types.UserSpacesReq) ([]types.Space, int, error)
+ UserLikesSpaces(ctx context.Context, req *types.UserSpacesReq, userID int64) ([]types.Space, int, error)
+ ListByPath(ctx context.Context, paths []string) ([]*types.Space, error)
+ AllowCallApi(ctx context.Context, spaceID int64, username string) (bool, error)
+ Delete(ctx context.Context, namespace, name, currentUser string) error
+ Deploy(ctx context.Context, namespace, name, currentUser string) (int64, error)
+ Wakeup(ctx context.Context, namespace, name string) error
+ Stop(ctx context.Context, namespace, name string, deleteSpace bool) error
+ // FixHasEntryFile checks whether git repo has entry point file and update space's HasAppFile property in db
+ FixHasEntryFile(ctx context.Context, s *database.Space) *database.Space
+ Status(ctx context.Context, namespace, name string) (string, string, error)
+ Logs(ctx context.Context, namespace, name string) (*deploy.MultiLogReader, error)
+ // HasEntryFile checks whether space repo has entry point file to run with
+ HasEntryFile(ctx context.Context, space *database.Space) bool
+}
+
+func NewSpaceComponent(config *config.Config) (SpaceComponent, error) {
+ c := &spaceComponentImpl{}
+ c.spaceStore = database.NewSpaceStore()
+ var err error
+ c.spaceSdkStore = database.NewSpaceSdkStore()
+ c.spaceResourceStore = database.NewSpaceResourceStore()
+ c.repoStore = database.NewRepoStore()
+ c.repoComponent, err = NewRepoComponentImpl(config)
+ if err != nil {
+ return nil, err
+ }
+ c.deployer = deploy.NewDeployer()
+ c.publicRootDomain = config.Space.PublicRootDomain
+ c.userStore = database.NewUserStore()
+ c.accountingComponent, err = NewAccountingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.userResourcesStore = database.NewUserResourcesStore()
+ c.serverBaseUrl = config.APIServer.PublicDomain
+ c.userLikesStore = database.NewUserLikesStore()
+ c.config = config
+ c.userSvcClient = rpc.NewUserSvcHttpClient(fmt.Sprintf("%s:%d", config.User.Host, config.User.Port),
+ rpc.AuthWithApiKey(config.APIToken))
+
+ c.deployTaskStore = database.NewDeployTaskStore()
+ c.git, err = git.NewGitServer(config)
+ if err != nil {
+ return nil, err
+ }
+ return c, nil
+}
+
+type spaceComponentImpl struct {
+ repoComponent RepoComponent
+ git gitserver.GitServer
+ spaceStore database.SpaceStore
+ spaceSdkStore database.SpaceSdkStore
+ spaceResourceStore database.SpaceResourceStore
+ repoStore database.RepoStore
+ userStore database.UserStore
+ deployer deploy.Deployer
+ publicRootDomain string
+ accountingComponent AccountingComponent
+ userResourcesStore database.UserResourcesStore
+ serverBaseUrl string
+ userLikesStore database.UserLikesStore
+ config *config.Config
+ userSvcClient rpc.UserSvcClient
+ deployTaskStore database.DeployTaskStore
+}
+
+func (c *spaceComponentImpl) Create(ctx context.Context, req types.CreateSpaceReq) (*types.Space, error) {
+ var nickname string
+ if req.Nickname != "" {
+ nickname = req.Nickname
+ } else {
+ nickname = req.Name
+ }
+
+ if req.DefaultBranch == "" {
+ req.DefaultBranch = types.MainBranch
+ }
+
+ req.Nickname = nickname
+ req.RepoType = types.SpaceRepo
+ req.Readme = generateReadmeData(req.License)
+ resource, err := c.spaceResourceStore.FindByID(ctx, req.ResourceID)
+ if err != nil {
+ return nil, fmt.Errorf("fail to find resource by id, %w", err)
+ }
+ // check resource price data
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)),
+ ResourceID: strconv.FormatInt(int64(resource.ID), 10),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("fail to find price data, %w", err)
+ }
+ if priceData.Total == 0 {
+ return nil, fmt.Errorf("cannot find valid price data")
+ }
+ if priceData.Prices[0].SkuPrice > 0 && req.OrderDetailID == 0 {
+ // check balance
+ account, err := c.accountingComponent.QueryBalanceByUserIDInternal(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find user balance, %w", err)
+ }
+ if account.Balance <= 0 {
+ return nil, fmt.Errorf("balance is not enough to run fee resources. current balance: %f", account.Balance)
+ }
+ }
+ var hardware types.HardWare
+ err = json.Unmarshal([]byte(resource.Resources), &hardware)
+ if err != nil {
+ return nil, fmt.Errorf("invalid hardware setting, %w", err)
+ }
+ _, err = c.deployer.CheckResourceAvailable(ctx, req.ClusterID, req.OrderDetailID, &hardware)
+ if err != nil {
+ return nil, fmt.Errorf("fail to check resource, %w", err)
+ }
+
+ _, dbRepo, err := c.repoComponent.CreateRepo(ctx, req.CreateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ dbSpace := database.Space{
+ RepositoryID: dbRepo.ID,
+ Sdk: req.Sdk,
+ SdkVersion: req.SdkVersion,
+ CoverImageUrl: req.CoverImageUrl,
+ Env: req.Env,
+ Hardware: resource.Resources,
+ Secrets: req.Secrets,
+ SKU: strconv.FormatInt(resource.ID, 10),
+ OrderDetailID: req.OrderDetailID,
+ }
+
+ resSpace, err := c.spaceStore.Create(ctx, dbSpace)
+ if err != nil {
+ slog.Error("fail to create space in db", slog.Any("req", req), slog.String("error", err.Error()))
+ return nil, fmt.Errorf("fail to create space in db, error: %w", err)
+ }
+
+ // Create README.md file
+ err = c.git.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: dbRepo.User.Username,
+ Email: dbRepo.User.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: req.Readme,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: readmeFileName,
+ }, types.SpaceRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create README.md file, cause: %w", err)
+ }
+
+ // Create .gitattributes file
+ err = c.git.CreateRepoFile(buildCreateFileReq(&types.CreateFileParams{
+ Username: dbRepo.User.Username,
+ Email: dbRepo.User.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: spaceGitattributesContent,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: gitattributesFileName,
+ }, types.SpaceRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create .gitattributes file, cause: %w", err)
+ }
+
+ if req.Sdk == scheduler.STREAMLIT.Name {
+ // create .streamlit/config.toml for support cors
+ fileReq := types.CreateFileParams{
+ Username: dbRepo.User.Username,
+ Email: dbRepo.User.Email,
+ Message: initCommitMessage,
+ Branch: req.DefaultBranch,
+ Content: streamlitConfigContent,
+ NewBranch: req.DefaultBranch,
+ Namespace: req.Namespace,
+ Name: req.Name,
+ FilePath: streamlitConfig,
+ }
+ err = c.git.CreateRepoFile(buildCreateFileReq(&fileReq, types.SpaceRepo))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create .streamlit/config.toml file, cause: %w", err)
+ }
+ }
+
+ space := &types.Space{
+ Creator: req.Username,
+ License: req.License,
+ Path: dbRepo.Path,
+ Name: req.Name,
+ Sdk: req.Sdk,
+ SdkVersion: req.SdkVersion,
+ Env: req.Env,
+ Hardware: resource.Resources,
+ Secrets: req.Secrets,
+ CoverImageUrl: resSpace.CoverImageUrl,
+ Endpoint: "",
+ Status: "",
+ Private: req.Private,
+ CreatedAt: resSpace.CreatedAt,
+ }
+ return space, nil
+}
+
+func (c *spaceComponentImpl) Show(ctx context.Context, namespace, name, currentUser string) (*types.Space, error) {
+ var tags []types.RepoTag
+ space, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find space, error: %w", err)
+ }
+
+ permission, err := c.repoComponent.GetUserRepoPermission(ctx, currentUser, space.Repository)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user repo permission, error: %w", err)
+ }
+ if !permission.CanRead {
+ return nil, ErrUnauthorized
+ }
+
+ ns, err := c.repoComponent.GetNameSpaceInfo(ctx, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get namespace info for model, error: %w", err)
+ }
+
+ var endpoint string
+ svcName, status, _ := c.status(ctx, space)
+ if len(svcName) > 0 {
+ if c.publicRootDomain == "" {
+ if space.Sdk == scheduler.STREAMLIT.Name || space.Sdk == scheduler.GRADIO.Name {
+ // if endpoint not ends with /, fastapi based app (gradio and streamlit) will redirect with http 307
+ // see issue: https://stackoverflow.com/questions/70351360/keep-getting-307-temporary-redirect-before-returning-status-200-hosted-on-fast
+ endpoint, _ = url.JoinPath(c.serverBaseUrl, "endpoint", svcName, "/")
+ } else {
+ endpoint, _ = url.JoinPath(c.serverBaseUrl, "endpoint", svcName)
+ }
+ endpoint = strings.Replace(endpoint, "http://", "", 1)
+ endpoint = strings.Replace(endpoint, "https://", "", 1)
+ } else {
+ endpoint = fmt.Sprintf("%s.%s", svcName, c.publicRootDomain)
+ }
+ }
+
+ likeExists, err := c.userLikesStore.IsExist(ctx, currentUser, space.Repository.ID)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user likes,error:%w", err)
+ return nil, newError
+ }
+ repository := common.BuildCloneInfo(c.config, space.Repository)
+
+ resModel := &types.Space{
+ ID: space.ID,
+ Name: space.Repository.Name,
+ Nickname: space.Repository.Nickname,
+ Description: space.Repository.Description,
+ Likes: space.Repository.Likes,
+ Path: space.Repository.Path,
+ License: space.Repository.License,
+ DefaultBranch: space.Repository.DefaultBranch,
+ Repository: &repository,
+ Private: space.Repository.Private,
+ Tags: tags,
+ User: &types.User{
+ Username: space.Repository.User.Username,
+ Nickname: space.Repository.User.NickName,
+ Email: space.Repository.User.Email,
+ },
+ CreatedAt: space.CreatedAt,
+ UpdatedAt: space.Repository.UpdatedAt,
+ Status: status,
+ Endpoint: endpoint,
+ Hardware: space.Hardware,
+ RepositoryID: space.Repository.ID,
+ UserLikes: likeExists,
+ Sdk: space.Sdk,
+ SdkVersion: space.SdkVersion,
+ CoverImageUrl: space.CoverImageUrl,
+ Source: space.Repository.Source,
+ SyncStatus: space.Repository.SyncStatus,
+ SKU: space.SKU,
+ SvcName: svcName,
+ CanWrite: permission.CanWrite,
+ CanManage: permission.CanAdmin,
+ Namespace: ns,
+ }
+ if permission.CanAdmin {
+ resModel.SensitiveCheckStatus = space.Repository.SensitiveCheckStatus.String()
+ }
+
+ return resModel, nil
+}
+
+func (c *spaceComponentImpl) Update(ctx context.Context, req *types.UpdateSpaceReq) (*types.Space, error) {
+ req.RepoType = types.SpaceRepo
+ if req.ResourceID != nil {
+ resource, err := c.spaceResourceStore.FindByID(ctx, *req.ResourceID)
+ if err != nil {
+ return nil, fmt.Errorf("fail to find resource by id, %w", err)
+ }
+
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUPayAsYouGo)),
+ ResourceID: strconv.FormatInt(int64(resource.ID), 10),
+ })
+ if err != nil {
+ return nil, fmt.Errorf("fail to find price data, %w", err)
+ }
+ if priceData.Total == 0 {
+ return nil, fmt.Errorf("cannot find valid price data")
+ }
+ if priceData.Prices[0].SkuPrice > 0 {
+ // check balance
+ account, err := c.accountingComponent.QueryBalanceByUserIDInternal(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("cannot find user balance, %w", err)
+ }
+ if account.Balance <= 0 {
+ return nil, fmt.Errorf("balance is not enough to run fee resources. current balance: %f", account.Balance)
+ }
+ }
+ }
+ dbRepo, err := c.repoComponent.UpdateRepo(ctx, req.UpdateRepoReq)
+ if err != nil {
+ return nil, err
+ }
+
+ space, err := c.spaceStore.ByRepoID(ctx, dbRepo.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find space, error: %w", err)
+ }
+ // don't support switch reserved resource
+ if space.OrderDetailID != 0 && req.ResourceID != nil {
+ return nil, fmt.Errorf("don't support switch reserved resource so far")
+ }
+ err = c.mergeUpdateSpaceRequest(ctx, space, req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to merge update space request, error: %w", err)
+ }
+
+ err = c.spaceStore.Update(ctx, *space)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update database space, error: %w", err)
+ }
+
+ resDataset := &types.Space{
+ ID: space.ID,
+ Name: dbRepo.Name,
+ Path: dbRepo.Path,
+ Sdk: space.Sdk,
+ SdkVersion: space.SdkVersion,
+ Template: space.Template,
+ Env: space.Env,
+ Hardware: space.Hardware,
+ Secrets: space.Secrets,
+ CoverImageUrl: space.CoverImageUrl,
+ License: dbRepo.License,
+ Private: dbRepo.Private,
+ CreatedAt: dbRepo.CreatedAt,
+ SKU: space.SKU,
+ }
+
+ return resDataset, nil
+}
+
+func (c *spaceComponentImpl) Index(ctx context.Context, filter *types.RepoFilter, per, page int) ([]types.Space, int, error) {
+ var (
+ resSpaces []types.Space
+ err error
+ )
+ repos, total, err := c.repoComponent.PublicToUser(ctx, types.SpaceRepo, filter.Username, filter, per, page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get public space repos,error:%w", err)
+ return nil, 0, newError
+ }
+ var repoIDs []int64
+ for _, repo := range repos {
+ repoIDs = append(repoIDs, repo.ID)
+ }
+ spaces, err := c.spaceStore.ByRepoIDs(ctx, repoIDs)
+ if err != nil {
+ newError := fmt.Errorf("failed to get spaces by repo ids,error:%w", err)
+ return nil, 0, newError
+ }
+
+ // loop through repos to keep the repos in sort order
+ for _, repo := range repos {
+ var space *database.Space
+ for _, s := range spaces {
+ if s.RepositoryID == repo.ID {
+ space = &s
+ space.Repository = repo
+ break
+ }
+ }
+ if space == nil {
+ continue
+ }
+ _, status, _ := c.status(ctx, space)
+ var tags []types.RepoTag
+ for _, tag := range space.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ resSpaces = append(resSpaces, types.Space{
+ Name: space.Repository.Name,
+ Description: space.Repository.Description,
+ Path: space.Repository.Path,
+ Sdk: space.Sdk,
+ SdkVersion: space.SdkVersion,
+ Template: space.Template,
+ Env: space.Env,
+ Hardware: space.Hardware,
+ Secrets: space.Secrets,
+ CoverImageUrl: space.CoverImageUrl,
+ License: space.Repository.License,
+ Private: space.Repository.Private,
+ Likes: space.Repository.Likes,
+ CreatedAt: space.Repository.CreatedAt,
+ UpdatedAt: space.Repository.UpdatedAt,
+ Tags: tags,
+ Status: status,
+ RepositoryID: space.Repository.ID,
+ Source: repo.Source,
+ SyncStatus: repo.SyncStatus,
+ })
+ }
+ return resSpaces, total, nil
+}
+
+func (c *spaceComponentImpl) OrgSpaces(ctx context.Context, req *types.OrgSpacesReq) ([]types.Space, int, error) {
+ var resSpaces []types.Space
+ var err error
+ r := membership.RoleUnknown
+ if req.CurrentUser != "" {
+ r, err = c.userSvcClient.GetMemberRole(ctx, req.Namespace, req.CurrentUser)
+ // log error, and treat user as unknown role in org
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Namespace), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ }
+ onlyPublic := !r.CanRead()
+ spaces, total, err := c.spaceStore.ByOrgPath(ctx, req.Namespace, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get org spaces,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range spaces {
+ _, status, _ := c.status(ctx, &data)
+ resSpaces = append(resSpaces, types.Space{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Path: data.Repository.Path,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ RepositoryID: data.Repository.ID,
+ CoverImageUrl: data.CoverImageUrl,
+ Status: status,
+ })
+ }
+
+ return resSpaces, total, nil
+}
+
+// UserSpaces get spaces of owner and visible to current user
+func (c *spaceComponentImpl) UserSpaces(ctx context.Context, req *types.UserSpacesReq) ([]types.Space, int, error) {
+ onlyPublic := req.Owner != req.CurrentUser
+ ms, total, err := c.spaceStore.ByUsername(ctx, req.Owner, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get spaces by username,%w", err)
+ return nil, 0, newError
+ }
+
+ var resSpaces []types.Space
+ for _, data := range ms {
+ _, status, _ := c.status(ctx, &data)
+ resSpaces = append(resSpaces, types.Space{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ Hardware: data.Hardware,
+ Status: status,
+ CoverImageUrl: data.CoverImageUrl,
+ })
+ }
+
+ return resSpaces, total, nil
+}
+
+func (c *spaceComponentImpl) UserLikesSpaces(ctx context.Context, req *types.UserSpacesReq, userID int64) ([]types.Space, int, error) {
+ ms, total, err := c.spaceStore.ByUserLikes(ctx, userID, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get spaces by username,%w", err)
+ return nil, 0, newError
+ }
+
+ var resSpaces []types.Space
+ for _, data := range ms {
+ _, status, _ := c.status(ctx, &data)
+ resSpaces = append(resSpaces, types.Space{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Path: data.Repository.Path,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ Hardware: data.Hardware,
+ Status: status,
+ CoverImageUrl: data.CoverImageUrl,
+ })
+ }
+
+ return resSpaces, total, nil
+}
+
+func (c *spaceComponentImpl) ListByPath(ctx context.Context, paths []string) ([]*types.Space, error) {
+ var spaces []*types.Space
+
+ spacesData, err := c.spaceStore.ListByPath(ctx, paths)
+ if err != nil {
+ return nil, fmt.Errorf("list space db failed, %w", err)
+ }
+ for _, data := range spacesData {
+ _, status, _ := c.status(ctx, &data)
+ var tags []types.RepoTag
+ for _, tag := range data.Repository.Tags {
+ tags = append(tags, types.RepoTag{
+ Name: tag.Name,
+ Category: tag.Category,
+ Group: tag.Group,
+ BuiltIn: tag.BuiltIn,
+ ShowName: tag.ShowName,
+ CreatedAt: tag.CreatedAt,
+ UpdatedAt: tag.UpdatedAt,
+ })
+ }
+ spaces = append(spaces, &types.Space{
+ Name: data.Repository.Name,
+ Description: data.Repository.Description,
+ Path: data.Repository.Path,
+ Sdk: data.Sdk,
+ SdkVersion: data.SdkVersion,
+ Template: data.Template,
+ Env: data.Env,
+ Hardware: data.Hardware,
+ Secrets: data.Secrets,
+ CoverImageUrl: data.CoverImageUrl,
+ License: data.Repository.License,
+ Private: data.Repository.Private,
+ Likes: data.Repository.Likes,
+ CreatedAt: data.Repository.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ Tags: tags,
+ Status: status,
+ RepositoryID: data.Repository.ID,
+ })
+ }
+ return spaces, nil
+}
+
+func (c *spaceComponentImpl) AllowCallApi(ctx context.Context, spaceID int64, username string) (bool, error) {
+ if username == "" {
+ return false, ErrUserNotFound
+ }
+ s, err := c.spaceStore.ByID(ctx, spaceID)
+ if err != nil {
+ return false, fmt.Errorf("failed to get space by id:%d, %w", spaceID, err)
+ }
+ fields := strings.Split(s.Repository.Path, "/")
+ return c.repoComponent.AllowReadAccess(ctx, s.Repository.RepositoryType, fields[0], fields[1], username)
+}
+
+func (c *spaceComponentImpl) Delete(ctx context.Context, namespace, name, currentUser string) error {
+ space, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find space, error: %w", err)
+ }
+
+ deleteDatabaseRepoReq := types.DeleteRepoReq{
+ Username: currentUser,
+ Namespace: namespace,
+ Name: name,
+ RepoType: types.SpaceRepo,
+ }
+ _, err = c.repoComponent.DeleteRepo(ctx, deleteDatabaseRepoReq)
+ if err != nil {
+ return fmt.Errorf("failed to delete repo of space, error: %w", err)
+ }
+
+ err = c.spaceStore.Delete(ctx, *space)
+ if err != nil {
+ return fmt.Errorf("failed to delete database space, error: %w", err)
+ }
+
+ // stop any running space instance
+ go func() {
+ err := c.Stop(ctx, namespace, name, true)
+ if err != nil {
+ slog.Error("stop space failed", slog.Any("error", err))
+ }
+ }()
+
+ return nil
+}
+
+func (c *spaceComponentImpl) Deploy(ctx context.Context, namespace, name, currentUser string) (int64, error) {
+ s, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ slog.Error("can't deploy space", slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name))
+ return -1, err
+ }
+ // found user id
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ slog.Error("can't find user for create deploy space", slog.Any("error", err), slog.String("username", currentUser))
+ return -1, err
+ }
+
+ // put repo-type and namespace/name in annotation
+ annotations := make(map[string]string)
+ annotations[types.ResTypeKey] = string(types.SpaceRepo)
+ annotations[types.ResNameKey] = fmt.Sprintf("%s/%s", namespace, name)
+ annoStr, err := json.Marshal(annotations)
+ if err != nil {
+ slog.Error("fail to create annotations for deploy space", slog.Any("error", err), slog.String("username", currentUser))
+ return -1, err
+ }
+
+ containerImg := ""
+ slog.Info("get space for deploy", slog.Any("space", s), slog.Any("NGINX", scheduler.NGINX))
+ slog.Info("compare space sdk", slog.Any("s.Sdk", s.Sdk), slog.Any("scheduler.NGINX.Name", scheduler.NGINX.Name))
+ if s.Sdk == scheduler.NGINX.Name {
+ slog.Warn("space use nginx pre-define image", slog.Any("namespace", namespace), slog.Any("name", name), slog.Any("scheduler.NGINX.Image", scheduler.NGINX.Image))
+ // Use default image for nginx space
+ containerImg = scheduler.NGINX.Image
+ }
+ slog.Info("run space with container image", slog.Any("namespace", namespace), slog.Any("name", name), slog.Any("containerImg", containerImg))
+
+ // create deploy for space
+ return c.deployer.Deploy(ctx, types.DeployRepo{
+ SpaceID: s.ID,
+ Path: s.Repository.Path,
+ GitPath: s.Repository.GitPath,
+ GitBranch: s.Repository.DefaultBranch,
+ Sdk: s.Sdk,
+ SdkVersion: s.SdkVersion,
+ Template: s.Template,
+ Env: s.Env,
+ Hardware: s.Hardware,
+ Secret: s.Secrets,
+ RepoID: s.Repository.ID,
+ ModelID: 0,
+ UserID: user.ID,
+ Annotation: string(annoStr),
+ ImageID: containerImg,
+ Type: types.SpaceType,
+ UserUUID: user.UUID,
+ SKU: s.SKU,
+ OrderDetailID: s.OrderDetailID,
+ })
+}
+
+func (c *spaceComponentImpl) Wakeup(ctx context.Context, namespace, name string) error {
+ s, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ slog.Error("can't wakeup space", slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name))
+ return err
+ }
+ // get latest Deploy for space
+ deploy, err := c.deployTaskStore.GetLatestDeployBySpaceID(ctx, s.ID)
+ if err != nil {
+ return fmt.Errorf("can't get space delopyment,%w", err)
+ }
+ return c.deployer.Wakeup(ctx, types.DeployRepo{
+ SpaceID: s.ID,
+ Namespace: namespace,
+ Name: name,
+ SvcName: deploy.SvcName,
+ })
+}
+
+func (c *spaceComponentImpl) Stop(ctx context.Context, namespace, name string, deleteSpace bool) error {
+ s, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ slog.Error("can't stop space", slog.Any("error", err), slog.String("namespace", namespace), slog.String("name", name))
+ return err
+ }
+ // get latest Deploy of space
+ deploy, err := c.deployTaskStore.GetLatestDeployBySpaceID(ctx, s.ID)
+ if err != nil {
+ slog.Error("can't get space deployment", slog.Any("error", err), slog.Any("space id", s.ID))
+ return fmt.Errorf("can't get space deployment,%w", err)
+ }
+ if deploy == nil {
+ return fmt.Errorf("can't get space deployment")
+ }
+
+ err = c.deployer.Stop(ctx, types.DeployRepo{
+ SpaceID: s.ID,
+ Namespace: namespace,
+ Name: name,
+ SvcName: deploy.SvcName,
+ OrderDetailID: deploy.OrderDetailID,
+ })
+ if err != nil {
+ return fmt.Errorf("can't stop space service deploy for service '%s', %w", deploy.SvcName, err)
+ }
+
+ err = c.deployTaskStore.StopDeploy(ctx, types.SpaceRepo, deploy.RepoID, deploy.UserID, deploy.ID)
+ if err != nil {
+ return fmt.Errorf("fail to update space deploy status to stopped for deploy ID '%d', %w", deploy.ID, err)
+ }
+ return nil
+}
+
+// FixHasEntryFile checks whether git repo has entry point file and update space's HasAppFile property in db
+func (c *spaceComponentImpl) FixHasEntryFile(ctx context.Context, s *database.Space) *database.Space {
+ hasAppFile := c.HasEntryFile(ctx, s)
+ if s.HasAppFile != hasAppFile {
+ s.HasAppFile = hasAppFile
+ _ = c.spaceStore.Update(ctx, *s)
+ }
+
+ return s
+}
+
+func (c *spaceComponentImpl) status(ctx context.Context, s *database.Space) (string, string, error) {
+ if !s.HasAppFile {
+ if s.Sdk == scheduler.NGINX.Name {
+ return "", SpaceStatusNoNGINXConf, nil
+ }
+ return "", SpaceStatusNoAppFile, nil
+ }
+ // get latest Deploy for space by space id
+ deploy, err := c.deployTaskStore.GetLatestDeployBySpaceID(ctx, s.ID)
+ if err != nil || deploy == nil {
+ slog.Error("fail to get latest space deploy by space id", slog.Any("SpaceID", s.ID))
+ return "", SpaceStatusStopped, fmt.Errorf("can't get space deployment,%w", err)
+ }
+
+ namespace, name := s.Repository.NamespaceAndName()
+ // request space deploy status by deploy id
+ srvName, code, _, err := c.deployer.Status(ctx, types.DeployRepo{
+ DeployID: deploy.ID,
+ SpaceID: deploy.SpaceID,
+ ModelID: deploy.ModelID,
+ Namespace: namespace,
+ Name: name,
+ SvcName: deploy.SvcName,
+ }, true)
+ if err != nil {
+ slog.Error("error happen when get space status", slog.Any("error", err), slog.String("path", s.Repository.Path))
+ return "", SpaceStatusStopped, err
+ }
+ return srvName, deployStatusCodeToString(code), nil
+}
+
+func (c *spaceComponentImpl) Status(ctx context.Context, namespace, name string) (string, string, error) {
+ s, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return "", SpaceStatusStopped, fmt.Errorf("can't find space by path:%w", err)
+ }
+ return c.status(ctx, s)
+}
+
+func (c *spaceComponentImpl) Logs(ctx context.Context, namespace, name string) (*deploy.MultiLogReader, error) {
+ s, err := c.spaceStore.FindByPath(ctx, namespace, name)
+ if err != nil {
+ return nil, fmt.Errorf("can't find space by path:%w", err)
+ }
+ return c.deployer.Logs(ctx, types.DeployRepo{
+ SpaceID: s.ID,
+ Namespace: namespace,
+ Name: name,
+ })
+}
+
+// HasEntryFile checks whether space repo has entry point file to run with
+func (c *spaceComponentImpl) HasEntryFile(ctx context.Context, space *database.Space) bool {
+ namespace, name := space.Repository.NamespaceAndName()
+ entryFile := "app.py"
+ if space.Sdk == scheduler.NGINX.Name {
+ entryFile = "nginx.conf"
+ }
+
+ return c.hasEntryFile(ctx, namespace, name, entryFile)
+}
+
+func (c *spaceComponentImpl) hasEntryFile(ctx context.Context, namespace, name, entryFile string) bool {
+ var req gitserver.GetRepoInfoByPathReq
+ req.Namespace = namespace
+ req.Name = name
+ // root dir
+ req.Path = ""
+ req.RepoType = types.SpaceRepo
+ files, err := c.git.GetRepoFileTree(ctx, req)
+ if err != nil {
+ slog.Error("check repo app file existence failed", slog.Any("eror", err))
+ return false
+ }
+
+ for _, f := range files {
+ if f.Type == "file" && f.Path == entryFile {
+ return true
+ }
+ }
+
+ return false
+}
+
+func (c *spaceComponentImpl) mergeUpdateSpaceRequest(ctx context.Context, space *database.Space, req *types.UpdateSpaceReq) error {
+ // Do not update column value if request body do not have it
+ if req.Sdk != nil {
+ space.Sdk = *req.Sdk
+ }
+ if req.SdkVersion != nil {
+ space.SdkVersion = *req.SdkVersion
+ }
+ if req.Env != nil {
+ space.Env = *req.Env
+ }
+ if req.Secrets != nil {
+ space.Secrets = *req.Secrets
+ }
+ if req.Template != nil {
+ space.Template = *req.Template
+ }
+ if req.CoverImageUrl != nil {
+ space.CoverImageUrl = *req.CoverImageUrl
+ }
+
+ if req.ResourceID != nil {
+ resource, err := c.spaceResourceStore.FindByID(ctx, *req.ResourceID)
+ if err != nil {
+ return fmt.Errorf("can't find space resource by id, resource id:%d, error:%w", *req.ResourceID, err)
+ }
+ space.Hardware = resource.Resources
+ space.SKU = strconv.FormatInt(resource.ID, 10)
+ }
+
+ return nil
+}
+
+const (
+ // SpaceStatusEmpty is the init status by default
+ SpaceStatusEmpty = ""
+ SpaceStatusBuilding = "Building"
+ SpaceStatusBuildFailed = "BuildingFailed"
+ SpaceStatusDeploying = "Deploying"
+ SpaceStatusDeployFailed = "DeployFailed"
+ SpaceStatusRunning = "Running"
+ SpaceStatusRuntimeError = "RuntimeError"
+ SpaceStatusStopped = "Stopped"
+ SpaceStatusSleeping = "Sleeping"
+
+ SpaceStatusNoAppFile = "NoAppFile"
+ RepoStatusDeleted = "Deleted"
+ SpaceStatusNoNGINXConf = "NoNGINXConf"
+)
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "strconv"
+
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type SpaceResourceComponent interface {
+ Index(ctx context.Context, clusterId string, deployType int, currentUser string) ([]types.SpaceResource, error)
+ Update(ctx context.Context, req *types.UpdateSpaceResourceReq) (*types.SpaceResource, error)
+ Create(ctx context.Context, req *types.CreateSpaceResourceReq) (*types.SpaceResource, error)
+ Delete(ctx context.Context, id int64) error
+}
+
+func NewSpaceResourceComponent(config *config.Config) (SpaceResourceComponent, error) {
+ c := &spaceResourceComponentImpl{}
+ c.spaceResourceStore = database.NewSpaceResourceStore()
+ c.deployer = deploy.NewDeployer()
+ c.userResourceStore = database.NewUserResourcesStore()
+ c.userStore = database.NewUserStore()
+ ac, err := NewAccountingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.accountComponent = ac
+ return c, nil
+}
+
+type spaceResourceComponentImpl struct {
+ spaceResourceStore database.SpaceResourceStore
+ deployer deploy.Deployer
+ userResourceStore database.UserResourcesStore
+ userStore database.UserStore
+ accountComponent AccountingComponent
+}
+
+func (c *spaceResourceComponentImpl) Index(ctx context.Context, clusterId string, deployType int, currentUser string) ([]types.SpaceResource, error) {
+ // backward compatibility for old api
+ if clusterId == "" {
+ clusters, err := c.deployer.ListCluster(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if len(clusters) == 0 {
+ return nil, fmt.Errorf("can not list clusters")
+ }
+ clusterId = clusters[0].ClusterID
+ }
+ var result []types.SpaceResource
+ databaseSpaceResources, err := c.spaceResourceStore.Index(ctx, clusterId)
+ if err != nil {
+ return nil, err
+ }
+ clusterResources, err := c.deployer.GetClusterById(ctx, clusterId)
+ if err != nil {
+ return nil, err
+ }
+ req := types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ SkuKind: strconv.Itoa(int(types.SKUTimeSpan)),
+ }
+ priceData, err := c.accountComponent.QueryPricesBySKUType(currentUser, req)
+ if err != nil {
+ return nil, err
+ }
+ for _, r := range databaseSpaceResources {
+ var isAvailable bool
+ var hardware types.HardWare
+ err := json.Unmarshal([]byte(r.Resources), &hardware)
+ if err != nil {
+ slog.Error("invalid hardware setting", slog.Any("error", err), slog.String("hardware", r.Resources))
+ } else {
+ isAvailable = deploy.CheckResource(clusterResources, &hardware)
+ }
+ if deployType == types.FinetuneType {
+ if hardware.Gpu.Num == "" && hardware.Npu.Num == "" {
+ continue
+ }
+ }
+ resourceType := types.ResourceTypeCPU
+ if hardware.Gpu.Num != "" {
+ resourceType = types.ResourceTypeGPU
+ } else if hardware.Npu.Num != "" {
+ resourceType = types.ResourceTypeNPU
+ }
+ price := getPrice(strconv.Itoa(int(r.ID)), priceData.Prices)
+ payMode := types.PayModeMinute
+ if price == 0 {
+ payMode = types.PayModeFree
+ }
+ result = append(result, types.SpaceResource{
+ ID: r.ID,
+ Name: r.Name,
+ Resources: r.Resources,
+ Price: price,
+ IsAvailable: isAvailable,
+ Type: resourceType,
+ PayMode: payMode,
+ })
+ }
+ // append user's resource
+ if currentUser != "" {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ return nil, newError
+ }
+ userResources, err := c.userResourceStore.GetReservedUserResources(ctx, user.UUID, clusterId)
+ if err != nil {
+ return nil, err
+ }
+ for _, r := range userResources {
+ var hardware types.HardWare
+ _ = json.Unmarshal([]byte(r.SpaceResource.Resources), &hardware)
+ resourceType := types.ResourceTypeCPU
+ if hardware.Gpu.Num != "" {
+ resourceType = types.ResourceTypeGPU
+ } else if hardware.Npu.Num != "" {
+ resourceType = types.ResourceTypeNPU
+ }
+ result = append(result, types.SpaceResource{
+ ID: r.SpaceResource.ID,
+ Name: r.SpaceResource.Name,
+ Resources: r.SpaceResource.Resources,
+ Price: r.Price,
+ IsAvailable: true,
+ Type: resourceType,
+ PayMode: types.PayMode(r.PayMode),
+ IsReserved: true,
+ OrderDetailId: r.OrderDetailId,
+ })
+ }
+
+ }
+
+ return result, nil
+}
+
+// get resource price
+func getPrice(resourceId string, prices []database.AccountPrice) float64 {
+ for _, p := range prices {
+ if p.ResourceID == resourceId {
+ return float64(p.SkuPrice) / 100
+ }
+ }
+ return 0
+}
+
+func (c *spaceResourceComponentImpl) Update(ctx context.Context, req *types.UpdateSpaceResourceReq) (*types.SpaceResource, error) {
+ sr, err := c.spaceResourceStore.FindByID(ctx, req.ID)
+ if err != nil {
+ slog.Error("error getting space resource", slog.Any("error", err))
+ return nil, err
+ }
+ sr.Name = req.Name
+ sr.Resources = req.Resources
+
+ sr, err = c.spaceResourceStore.Update(ctx, *sr)
+ if err != nil {
+ slog.Error("error updating space resource", slog.Any("error", err))
+ return nil, err
+ }
+
+ result := &types.SpaceResource{
+ ID: sr.ID,
+ Name: sr.Name,
+ Resources: sr.Resources,
+ }
+
+ return result, nil
+}
+
+func (c *spaceResourceComponentImpl) Create(ctx context.Context, req *types.CreateSpaceResourceReq) (*types.SpaceResource, error) {
+ sr := database.SpaceResource{
+ Name: req.Name,
+ Resources: req.Resources,
+ ClusterID: req.ClusterID,
+ }
+ res, err := c.spaceResourceStore.Create(ctx, sr)
+ if err != nil {
+ slog.Error("error creating space resource", slog.Any("error", err))
+ return nil, err
+ }
+
+ result := &types.SpaceResource{
+ ID: res.ID,
+ Name: res.Name,
+ Resources: res.Resources,
+ }
+
+ return result, nil
+}
+
+func (c *spaceResourceComponentImpl) Delete(ctx context.Context, id int64) error {
+ sr, err := c.spaceResourceStore.FindByID(ctx, id)
+ if err != nil {
+ slog.Error("error finding space resource", slog.Any("error", err))
+ return err
+ }
+
+ err = c.spaceResourceStore.Delete(ctx, *sr)
+ if err != nil {
+ slog.Error("error deleting space resource", slog.Any("error", err))
+ return err
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type SpaceSdkComponent interface {
+ Index(ctx context.Context) ([]types.SpaceSdk, error)
+ Update(ctx context.Context, req *types.UpdateSpaceSdkReq) (*types.SpaceSdk, error)
+ Create(ctx context.Context, req *types.CreateSpaceSdkReq) (*types.SpaceSdk, error)
+ Delete(ctx context.Context, id int64) error
+}
+
+func NewSpaceSdkComponent(config *config.Config) (SpaceSdkComponent, error) {
+ c := &spaceSdkComponentImpl{}
+ c.spaceSdkStore = database.NewSpaceSdkStore()
+
+ return c, nil
+}
+
+type spaceSdkComponentImpl struct {
+ spaceSdkStore database.SpaceSdkStore
+}
+
+func (c *spaceSdkComponentImpl) Index(ctx context.Context) ([]types.SpaceSdk, error) {
+ var result []types.SpaceSdk
+ databaseSpaceSdks, err := c.spaceSdkStore.Index(ctx)
+ if err != nil {
+ return nil, err
+ }
+ for _, r := range databaseSpaceSdks {
+ result = append(result, types.SpaceSdk{
+ ID: r.ID,
+ Name: r.Name,
+ Version: r.Version,
+ })
+ }
+
+ return result, nil
+}
+
+func (c *spaceSdkComponentImpl) Update(ctx context.Context, req *types.UpdateSpaceSdkReq) (*types.SpaceSdk, error) {
+ ss, err := c.spaceSdkStore.FindByID(ctx, req.ID)
+ if err != nil {
+ slog.Error("error getting space sdk", slog.Any("error", err))
+ return nil, err
+ }
+ ss.Name = req.Name
+ ss.Version = req.Version
+
+ ss, err = c.spaceSdkStore.Update(ctx, *ss)
+ if err != nil {
+ slog.Error("error getting space sdk", slog.Any("error", err))
+ return nil, err
+ }
+
+ result := &types.SpaceSdk{
+ ID: ss.ID,
+ Name: ss.Name,
+ Version: ss.Version,
+ }
+
+ return result, nil
+}
+
+func (c *spaceSdkComponentImpl) Create(ctx context.Context, req *types.CreateSpaceSdkReq) (*types.SpaceSdk, error) {
+ ss := database.SpaceSdk{
+ Name: req.Name,
+ Version: req.Version,
+ }
+ res, err := c.spaceSdkStore.Create(ctx, ss)
+ if err != nil {
+ slog.Error("error creating space sdk", slog.Any("error", err))
+ return nil, err
+ }
+
+ result := &types.SpaceSdk{
+ ID: res.ID,
+ Name: res.Name,
+ Version: res.Version,
+ }
+
+ return result, nil
+}
+
+func (c *spaceSdkComponentImpl) Delete(ctx context.Context, id int64) error {
+ ss, err := c.spaceSdkStore.FindByID(ctx, id)
+ if err != nil {
+ slog.Error("error finding space sdk", slog.Any("error", err))
+ return err
+ }
+
+ err = c.spaceSdkStore.Delete(ctx, *ss)
+ if err != nil {
+ slog.Error("error deleting space sdk", slog.Any("error", err))
+ return err
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+)
+
+type SSHKeyComponent interface {
+ Create(ctx context.Context, req *types.CreateSSHKeyRequest) (*database.SSHKey, error)
+ Index(ctx context.Context, username string, per, page int) ([]database.SSHKey, error)
+ Delete(ctx context.Context, username, name string) error
+}
+
+func NewSSHKeyComponent(config *config.Config) (SSHKeyComponent, error) {
+ c := &sSHKeyComponentImpl{}
+ c.ss = database.NewSSHKeyStore()
+ c.us = database.NewUserStore()
+ var err error
+ c.gs, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("failed to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ return c, nil
+}
+
+type sSHKeyComponentImpl struct {
+ ss database.SSHKeyStore
+ us database.UserStore
+ gs gitserver.GitServer
+}
+
+func (c *sSHKeyComponentImpl) Create(ctx context.Context, req *types.CreateSSHKeyRequest) (*database.SSHKey, error) {
+ user, err := c.us.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user,error:%w", err)
+ }
+
+ nameExistsKey, err := c.ss.FindByNameAndUserID(ctx, req.Name, user.ID)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, fmt.Errorf("failed to find if ssh key exists,error:%w", err)
+ }
+ if nameExistsKey.ID != 0 {
+ return nil, fmt.Errorf("ssh key name already exists")
+ }
+
+ contentExistsKey, err := c.ss.FindByKeyContent(ctx, req.Content)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, fmt.Errorf("failed to find if ssh key exists,error:%w", err)
+ }
+ if contentExistsKey.ID != 0 {
+ return nil, fmt.Errorf("ssh key already exists")
+ }
+ sk, err := c.gs.CreateSSHKey(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git SSH key,error:%w", err)
+ }
+ fingerprint, err := common.CalculateSSHKeyFingerprint(req.Content)
+ if err != nil {
+ return nil, fmt.Errorf("failed to calculate ssh key fingerprint,error:%w", err)
+ }
+ if sk == nil {
+ sk = &database.SSHKey{
+ GitID: 0,
+ Name: req.Name,
+ Content: req.Content,
+ UserID: user.ID,
+ }
+ }
+ sk.UserID = user.ID
+ sk.FingerprintSHA256 = fingerprint
+ resSk, err := c.ss.Create(ctx, sk)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create database SSH key,error:%w", err)
+ }
+ return resSk, nil
+}
+
+func (c *sSHKeyComponentImpl) Index(ctx context.Context, username string, per, page int) ([]database.SSHKey, error) {
+ sks, err := c.ss.Index(ctx, username, per, page)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get database SSH keys,error:%w", err)
+ }
+ return sks, nil
+}
+
+func (c *sSHKeyComponentImpl) Delete(ctx context.Context, username, name string) error {
+ sshKey, err := c.ss.FindByUsernameAndName(ctx, username, name)
+ if err != nil {
+ return fmt.Errorf("failed to get database SSH keys,error:%w", err)
+ }
+ err = c.gs.DeleteSSHKey(int(sshKey.GitID))
+ if err != nil {
+ return fmt.Errorf("failed to delete git SSH keys,error:%w", err)
+ }
+ err = c.ss.Delete(ctx, sshKey.ID)
+ if err != nil {
+ return fmt.Errorf("failed to delete database SSH keys,error:%w", err)
+ }
+ return nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type syncClientSettingComponentImpl struct {
+ settingStore database.SyncClientSettingStore
+ userStore database.UserStore
+}
+
+type SyncClientSettingComponent interface {
+ Create(ctx context.Context, req types.CreateSyncClientSettingReq) (*database.SyncClientSetting, error)
+ Show(ctx context.Context, currentUser string) (*database.SyncClientSetting, error)
+}
+
+func NewSyncClientSettingComponent(config *config.Config) (SyncClientSettingComponent, error) {
+ return &syncClientSettingComponentImpl{
+ settingStore: database.NewSyncClientSettingStore(),
+ userStore: database.NewUserStore(),
+ }, nil
+}
+
+func (c *syncClientSettingComponentImpl) Create(ctx context.Context, req types.CreateSyncClientSettingReq) (*database.SyncClientSetting, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !user.CanAdmin() {
+ return nil, fmt.Errorf("only admin was allowed create sync client setting")
+ }
+ exists, err := c.settingStore.SyncClientSettingExists(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check sync client setting if exists, error: %w", err)
+ }
+ if exists {
+ err := c.settingStore.DeleteAll(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to delete existing sync client setting, error: %w", err)
+ }
+ }
+ var mt database.SyncClientSetting
+ mt.Token = req.Token
+ mt.ConcurrentCount = req.ConcurrentCount
+ mt.MaxBandwidth = req.MaxBandwidth
+ res, err := c.settingStore.Create(ctx, &mt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create sync client setting, error: %w", err)
+ }
+ return res, nil
+}
+
+func (c *syncClientSettingComponentImpl) Show(ctx context.Context, currentUser string) (*database.SyncClientSetting, error) {
+ user, err := c.userStore.FindByUsername(ctx, currentUser)
+ if err != nil {
+ return nil, ErrUnauthorized
+ }
+ if !user.CanAdmin() {
+ return nil, fmt.Errorf("only admin was allowed get sync client setting")
+ }
+ res, err := c.settingStore.First(ctx)
+ if err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return nil, nil
+ }
+ return nil, fmt.Errorf("failed to create sync client setting, error: %w", err)
+ }
+ return res, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "slices"
+
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/component/tagparser"
+)
+
+type TagComponent interface {
+ AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error)
+ ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string) error
+ UpdateMetaTags(ctx context.Context, tagScope database.TagScope, namespace, name, content string) ([]*database.RepositoryTag, error)
+ UpdateLibraryTags(ctx context.Context, tagScope database.TagScope, namespace, name, oldFilePath, newFilePath string) error
+ UpdateRepoTagsByCategory(ctx context.Context, tagScope database.TagScope, repoID int64, category string, tagNames []string) error
+}
+
+func NewTagComponent(config *config.Config) (TagComponent, error) {
+ tc := &tagComponentImpl{}
+ tc.tagStore = database.NewTagStore()
+ tc.repoStore = database.NewRepoStore()
+ if config.SensitiveCheck.Enable {
+ tc.sensitiveChecker = rpc.NewModerationSvcHttpClient(fmt.Sprintf("%s:%d", config.Moderation.Host, config.Moderation.Port))
+ }
+ return tc, nil
+}
+
+type tagComponentImpl struct {
+ tagStore database.TagStore
+ repoStore database.RepoStore
+ sensitiveChecker rpc.ModerationSvcClient
+}
+
+func (tc *tagComponentImpl) AllTagsByScopeAndCategory(ctx context.Context, scope string, category string) ([]*database.Tag, error) {
+ return tc.tagStore.AllTagsByScopeAndCategory(ctx, database.TagScope(scope), category)
+}
+
+func (c *tagComponentImpl) ClearMetaTags(ctx context.Context, repoType types.RepositoryType, namespace, name string) error {
+
+ _, err := c.tagStore.SetMetaTags(ctx, repoType, namespace, name, nil)
+ return err
+}
+
+func (c *tagComponentImpl) UpdateMetaTags(ctx context.Context, tagScope database.TagScope, namespace, name, content string) ([]*database.RepositoryTag, error) {
+ var (
+ tp tagparser.TagProcessor
+ repoType types.RepositoryType
+ )
+ // TODO:load from cache
+
+ if tagScope == database.DatasetTagScope {
+ tp = tagparser.NewDatasetTagProcessor(c.tagStore)
+ repoType = types.DatasetRepo
+ } else if tagScope == database.ModelTagScope {
+ tp = tagparser.NewModelTagProcessor(c.tagStore)
+ repoType = types.ModelRepo
+ } else if tagScope == database.PromptTagScope {
+ tp = tagparser.NewPromptTagProcessor(c.tagStore)
+ repoType = types.PromptRepo
+ } else {
+ // skip tag process for code and space now
+ return nil, nil
+ }
+ tagsMatched, tagToCreate, err := tp.ProcessReadme(ctx, content)
+ if err != nil {
+ slog.Error("Failed to process tags", slog.Any("error", err))
+ return nil, fmt.Errorf("failed to process tags, cause: %w", err)
+ }
+
+ if c.sensitiveChecker != nil {
+ // TODO:do tag name sensitive checking in batch
+ // remove sensitive tags by checking tag name of tag to create
+ tagToCreate = slices.DeleteFunc(tagToCreate, func(t *database.Tag) bool {
+ result, err := c.sensitiveChecker.PassTextCheck(ctx, string(sensitive.ScenarioNicknameDetection), t.Name)
+ if err != nil {
+ slog.Error("Failed to check tag name sensitivity", slog.String("tag_name", t.Name), slog.Any("error", err))
+ return true
+ }
+ return result.IsSensitive
+ })
+ }
+
+ err = c.tagStore.SaveTags(ctx, tagToCreate)
+ if err != nil {
+ slog.Error("Failed to save tags", slog.Any("error", err))
+ return nil, fmt.Errorf("failed to save tags, cause: %w", err)
+ }
+
+ repo, err := c.repoStore.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ slog.Error("failed to find repo", slog.Any("error", err))
+ return nil, fmt.Errorf("failed to find repo, cause: %w", err)
+ }
+
+ metaTags := append(tagsMatched, tagToCreate...)
+ var repoTags []*database.RepositoryTag
+ repoTags, err = c.tagStore.SetMetaTags(ctx, repoType, namespace, name, metaTags)
+ if err != nil {
+ slog.Error("failed to set dataset's tags", slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ return nil, fmt.Errorf("failed to set dataset's tags, cause: %w", err)
+ }
+
+ err = c.repoStore.UpdateLicenseByTag(ctx, repo.ID)
+ if err != nil {
+ slog.Error("failed to update repo license tags", slog.Any("error", err))
+ }
+
+ return repoTags, nil
+}
+
+func (c *tagComponentImpl) UpdateLibraryTags(ctx context.Context, tagScope database.TagScope, namespace, name, oldFilePath, newFilePath string) error {
+ oldLibTagName := tagparser.LibraryTag(oldFilePath)
+ newLibTagName := tagparser.LibraryTag(newFilePath)
+ // TODO:load from cache
+ var (
+ allTags []*database.Tag
+ err error
+ repoType types.RepositoryType
+ )
+ if tagScope == database.DatasetTagScope {
+ allTags, err = c.tagStore.AllDatasetTags(ctx)
+ repoType = types.DatasetRepo
+ } else if tagScope == database.ModelTagScope {
+ allTags, err = c.tagStore.AllModelTags(ctx)
+ repoType = types.ModelRepo
+ } else if tagScope == database.PromptTagScope {
+ allTags, err = c.tagStore.AllPromptTags(ctx)
+ repoType = types.PromptRepo
+ } else {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("failed to get all tags, error: %w", err)
+ }
+ var oldLibTag, newLibTag *database.Tag
+ for _, t := range allTags {
+ if t.Category != "framework" {
+ continue
+ }
+ if t.Name == newLibTagName {
+ newLibTag = t
+ }
+ if t.Name == oldLibTagName {
+ oldLibTag = t
+ }
+ }
+ err = c.tagStore.SetLibraryTag(ctx, repoType, namespace, name, newLibTag, oldLibTag)
+ if err != nil {
+ slog.Error("failed to set %s's tags", string(repoType), slog.String("namespace", namespace),
+ slog.String("name", name), slog.Any("error", err))
+ return fmt.Errorf("failed to set Library tags, cause: %w", err)
+ }
+ return nil
+}
+
+func (c *tagComponentImpl) UpdateRepoTagsByCategory(ctx context.Context, tagScope database.TagScope, repoID int64, category string, tagNames []string) error {
+ allTags, err := c.tagStore.AllTagsByScopeAndCategory(ctx, tagScope, category)
+ if err != nil {
+ return fmt.Errorf("failed to get all tags of scope `%s`, error: %w", tagScope, err)
+ }
+
+ if len(allTags) == 0 {
+ return fmt.Errorf("no tags found for scope `%s` and category `%s`", tagScope, category)
+ }
+
+ var tagIDs []int64
+ for _, tagName := range tagNames {
+ for _, t := range allTags {
+ if t.Name == tagName {
+ tagIDs = append(tagIDs, t.ID)
+ }
+ }
+ }
+
+ var oldTagIDs []int64
+ oldTagIDs, err = c.repoStore.TagIDs(ctx, repoID, category)
+ if err != nil {
+ return fmt.Errorf("failed to get old tag ids, error: %w", err)
+ }
+ return c.tagStore.UpsertRepoTags(ctx, repoID, oldTagIDs, tagIDs)
+}
+
+
+
package tagparser
+
+import "strings"
+
+const (
+ categoryNameTask = "task"
+ categoryNameLicense = "license"
+ categoryNameFramework = "framework"
+ categoryNameSize = "size"
+ categoryNameLanguage = "language"
+)
+
+func formatCategoryName(name string) string {
+ switch {
+ //category task
+ case strings.EqualFold(name, categoryNameTask):
+ return categoryNameTask
+ case strings.EqualFold(name, "tasks"):
+ return categoryNameTask
+ case strings.EqualFold(name, "tags"):
+ return categoryNameTask
+ case strings.EqualFold(name, "task_categories"):
+ return categoryNameTask
+ case strings.EqualFold(name, "pipeline_tag"):
+ return categoryNameTask
+
+ //category license
+ case strings.EqualFold(name, categoryNameLicense):
+ return categoryNameLicense
+
+ //category framework
+ case strings.EqualFold(name, categoryNameFramework):
+ return categoryNameFramework
+
+ //category size
+ case strings.EqualFold(name, categoryNameSize):
+ return categoryNameSize
+ case strings.EqualFold(name, "size_categories"):
+ return categoryNameSize
+ default:
+ return name
+ }
+}
+
+
+
package tagparser
+
+import (
+ "log/slog"
+ "slices"
+ "strings"
+
+ "gopkg.in/yaml.v3"
+)
+
+// MetaTags parse metadata of README file, return tags found
+func MetaTags(readme string) (map[string][]string, error) {
+ meta := metaText(readme)
+ if len(meta) == 0 {
+ return map[string][]string{}, nil
+ }
+
+ categoryContents := make(map[string]any)
+ //parse yaml string
+ err := yaml.Unmarshal([]byte(meta), categoryContents)
+ if err != nil {
+ slog.Error("error unmarshall meta for tags", slog.Any("error", err), slog.String("meta", meta))
+ return nil, err
+ }
+ categoryTags := make(map[string][]string)
+ for category, content := range categoryContents {
+ category = formatCategoryName(category)
+ slog.Debug("tag category map content", slog.String("category", category), slog.Any("content", content))
+ //TODO: define different content parser
+ if tagName, match := content.(string); match {
+ categoryTags[category] = append(categoryTags[category], strings.TrimSpace(tagName))
+ } else if tagNames, match := content.([]interface{}); match {
+ for _, tagNameValue := range tagNames {
+ if tagName, isString := tagNameValue.(string); isString {
+ categoryTags[category] = append(categoryTags[category], strings.TrimSpace(tagName))
+ } else {
+ //ignore
+ slog.Warn("ignore unknown tag format", slog.Any("tagNameValue", tagNameValue))
+ }
+ }
+ } else {
+ slog.Error("Unknown meta format", slog.Any("content", content))
+ //alow continue parsing
+ // return nil, errors.New("unknown meta format")
+ }
+ }
+
+ return uniqueTags(categoryTags), nil
+}
+
+func uniqueTags(categoryTags map[string][]string) map[string][]string {
+ uniqueTags := make(map[string][]string)
+ for c, ts := range categoryTags {
+ //must sort before slice compact
+ slices.Sort(ts)
+ uniqueTags[c] = slices.Compact(ts)
+ }
+ return uniqueTags
+}
+
+func metaText(readme string) string {
+ splits := strings.Split(readme, "---")
+ if len(splits) < 2 {
+ return ""
+ }
+
+ return splits[1]
+}
+
+
+
package tagparser
+
+import (
+ "path/filepath"
+ "strings"
+)
+
+// LibraryTag parse file name or extension to match defined tag name
+// see: https://git-devops.opencsg.com/product/community/open-portal/-/issues/47
+func LibraryTag(filePath string) string {
+ if len(filePath) == 0 {
+ return ""
+ }
+ filename := filepath.Base(filePath)
+ filename = strings.ToLower(filename)
+ switch {
+ case isPytorch(filename):
+ return "pytorch"
+ case isTensorflow(filename):
+ return "tensorflow"
+ case isSafetensors(filename):
+ return "safetensors"
+ case isJAX(filename):
+ return "jax"
+ case strings.HasSuffix(filename, ".onnx"):
+ return "onnx"
+ case strings.HasSuffix(filename, ".pdparams"):
+ return "paddlepaddle"
+ case strings.HasSuffix(filename, ".joblib"):
+ return "joblib"
+ case strings.HasSuffix(filename, ".gguf"):
+ return "gguf"
+ default:
+ return ""
+ }
+}
+
+func isPytorch(filename string) bool {
+ return (strings.HasPrefix(filename, "pytorch_model") && strings.HasSuffix(filename, ".bin")) || strings.HasSuffix(filename, ".pt")
+}
+
+func isTensorflow(filename string) bool {
+ return strings.HasPrefix(filename, "tf_model") && strings.HasSuffix(filename, ".h5")
+}
+func isSafetensors(filename string) bool {
+ return strings.HasSuffix(filename, ".safetensors")
+}
+func isJAX(filename string) bool {
+ return strings.HasPrefix(filename, "flax_model") && strings.HasSuffix(filename, ".msgpack")
+}
+
+
+
package tagparser
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type DatasetTagStore interface {
+ AllDatasetTags(ctx context.Context) ([]*database.Tag, error)
+}
+
+type ModelTagStore interface {
+ AllModelTags(ctx context.Context) ([]*database.Tag, error)
+}
+
+type PromptTagStore interface {
+ AllPromptTags(ctx context.Context) ([]*database.Tag, error)
+}
+
+type TagProcessor interface {
+ ProcessReadme(ctx context.Context, content string) (tagsMatched, tagsNew []*database.Tag, err error)
+ ProcessFramework(ctx context.Context, fileName string) (*database.Tag, error)
+}
+
+// make sure tagProcessor implements TagProcessor
+var _ TagProcessor = (*tagProcessor)(nil)
+
+type tagProcessor struct {
+ existingTags func(ctx context.Context) ([]*database.Tag, error)
+ tagScope database.TagScope
+}
+
+func NewDatasetTagProcessor(ts DatasetTagStore) TagProcessor {
+ p := new(tagProcessor)
+ p.existingTags = ts.AllDatasetTags
+ p.tagScope = database.DatasetTagScope
+ return p
+}
+
+func NewModelTagProcessor(ts ModelTagStore) TagProcessor {
+ p := new(tagProcessor)
+ p.existingTags = ts.AllModelTags
+ p.tagScope = database.ModelTagScope
+ return p
+}
+
+func NewPromptTagProcessor(ts PromptTagStore) TagProcessor {
+ p := new(tagProcessor)
+ p.existingTags = ts.AllPromptTags
+ p.tagScope = database.PromptTagScope
+ return p
+}
+
+func (p *tagProcessor) ProcessReadme(ctx context.Context, content string) (tagsMatched, tagsNew []*database.Tag, err error) {
+ metaCategoryTags, err := MetaTags(content)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to parse metadata, cause: %w", err)
+ }
+ slog.Debug("File tags parsed", slog.Any("tags", metaCategoryTags))
+
+ var existingTags []*database.Tag
+ existingTags, err = p.existingTags(ctx)
+ if err != nil {
+ slog.Error("Failed to get exiting tags", slog.Any("error", err))
+ return nil, nil, fmt.Errorf("failed to get existing tags, cause: %w", err)
+ }
+
+ existingCategoryTags := p.mapCategoryTag(existingTags)
+ tagsMatched, tagsNew = p.processTags(existingCategoryTags, metaCategoryTags)
+ return
+}
+
+func (p *tagProcessor) ProcessFramework(ctx context.Context, fileName string) (*database.Tag, error) {
+ //TODO:move framework tag processing from component package to here
+ return nil, nil
+}
+
+// processTags compare tags input with existing tags, return tags matched and tags new
+func (p *tagProcessor) processTags(existingCategoryTagMap map[string]map[string]*database.Tag,
+ categoryTagMap map[string][]string) ([]*database.Tag, []*database.Tag) {
+ var tagsMatched []*database.Tag
+ var tagsToCreate []*database.Tag
+ for category, tagNames := range categoryTagMap {
+ existingTaskTags, found := existingCategoryTagMap[category]
+ if !found {
+ continue
+ }
+ for _, tagName := range tagNames {
+ if tag, ok := existingTaskTags[tagName]; !ok {
+ tagsToCreate = append(tagsToCreate, &database.Tag{
+ Name: tagName,
+ Category: category,
+ Scope: p.tagScope,
+ BuiltIn: false, // new tag is absolutely not built-in
+ Group: "", // keep empty
+ })
+ } else {
+ tagsMatched = append(tagsMatched, tag)
+ }
+ }
+
+ }
+
+ return tagsMatched, tagsToCreate
+}
+
+func (p *tagProcessor) mapCategoryTag(tags []*database.Tag) map[string]map[string]*database.Tag {
+ predefinedCategoryTagMap := make(map[string]map[string]*database.Tag)
+ for _, tag := range tags {
+ var ok bool
+ var tags map[string]*database.Tag
+ if tags, ok = predefinedCategoryTagMap[tag.Category]; !ok {
+ tags = make(map[string]*database.Tag)
+ predefinedCategoryTagMap[tag.Category] = tags
+ }
+ tags[tag.Name] = tag
+ }
+ return predefinedCategoryTagMap
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/types/telemetry"
+)
+
+type telemetryComponentImpl struct {
+ // Add telemetry related fields and methods here
+ ts database.TelemetryStore
+ us database.UserStore
+ rs database.RepoStore
+}
+
+type TelemetryComponent interface {
+ SaveUsageData(ctx context.Context, usage telemetry.Usage) error
+ GenUsageData(ctx context.Context) (telemetry.Usage, error)
+}
+
+func NewTelemetryComponent() (TelemetryComponent, error) {
+ ts := database.NewTelemetryStore()
+ us := database.NewUserStore()
+ rs := database.NewRepoStore()
+ return &telemetryComponentImpl{ts: ts, us: us, rs: rs}, nil
+}
+
+func (tc *telemetryComponentImpl) SaveUsageData(ctx context.Context, usage telemetry.Usage) error {
+ t := database.Telemetry{
+ UUID: usage.UUID,
+ RecordedAt: usage.RecordedAt,
+ Hostname: usage.Hostname,
+ Version: usage.Version,
+ InstallationType: usage.InstallationType,
+ ActiveUserCount: usage.ActiveUserCount,
+ Edition: usage.Edition,
+ LicenseMD5: usage.LicenseMD5,
+ LicenseID: usage.LicenseID,
+ HistoricalMaxUsers: usage.HistoricalMaxUsers,
+ Licensee: usage.Licensee,
+ LicenseUserCount: usage.LicenseUserCount,
+ LicenseBillableUsers: usage.LicenseBillableUsers,
+ LicenseStartsAt: usage.LicenseStartsAt,
+ LicenseExpiresAt: usage.LicenseExpiresAt,
+ LicensePlan: usage.LicensePlan,
+ LicenseAddOns: usage.LicenseAddOns,
+ Settings: usage.Settings,
+ Counts: usage.Counts,
+ }
+ err := tc.ts.Save(ctx, &t)
+ if err != nil {
+ return fmt.Errorf("failed to save telemetry data to db: %w", err)
+ }
+
+ return nil
+}
+
+func (tc *telemetryComponentImpl) GenUsageData(ctx context.Context) (telemetry.Usage, error) {
+ var usage telemetry.Usage
+
+ uuid, err := uuid.NewV7()
+ if err != nil {
+ return usage, fmt.Errorf("failed to generate uuid: %w", err)
+ }
+ usage.UUID = uuid.String()
+ usage.RecordedAt = time.Now()
+ usage.Version = ""
+ usage.InstallationType = ""
+ usage.ActiveUserCount, err = tc.getUserCnt(ctx)
+ if err != nil {
+ return usage, fmt.Errorf("failed to get user count: %w", err)
+ }
+ usage.Edition = ""
+ usage.HistoricalMaxUsers = 0
+ //TODO:load license data
+ // usage.LicenseMD5 = ""
+ // usage.LicenseID = 0
+ // usage.Licensee = telemetry.Licensee{}
+ // usage.LicenseUserCount = 0
+ // usage.LicenseBillableUsers = 0
+ // usage.LicenseStartsAt =
+ // usage.LicenseExpiresAt = ""
+ // usage.LicensePlan = ""
+ // usage.LicenseAddOns = ""
+ usage.Settings = telemetry.Settings{
+ // LdapEncryptedSecretsEnabled: false,
+ // SmtpEncryptedSecretsEnabled: false,
+ // OperatingSystem: "",
+ // GitalyApdex: 0,
+ // CollectedDataCategories: []string{},
+ // ServicePingFeaturesEnabled: false,
+ // SnowplowEnabled: false,
+ // SnowplowConfiguredToGitlabCollector: false,
+ }
+ usage.Counts, err = tc.getCounts(ctx)
+ if err != nil {
+ return usage, fmt.Errorf("failed to get counts: %w", err)
+ }
+ return usage, nil
+}
+
+func (tc *telemetryComponentImpl) getUserCnt(ctx context.Context) (int, error) {
+ return tc.us.CountUsers(ctx)
+}
+
+func (tc *telemetryComponentImpl) getCounts(ctx context.Context) (telemetry.Counts, error) {
+ var counts telemetry.Counts
+ modelCnt, err := tc.rs.CountByRepoType(ctx, types.ModelRepo)
+ if err != nil {
+ return counts, fmt.Errorf("failed to get model repo count: %w", err)
+ }
+
+ dsCnt, err := tc.rs.CountByRepoType(ctx, types.DatasetRepo)
+ if err != nil {
+ return counts, fmt.Errorf("failed to get dataset repo count: %w", err)
+ }
+
+ codeCnt, err := tc.rs.CountByRepoType(ctx, types.CodeRepo)
+ if err != nil {
+ return counts, fmt.Errorf("failed to get code repo count: %w", err)
+ }
+
+ spaceCnt, err := tc.rs.CountByRepoType(ctx, types.SpaceRepo)
+ if err != nil {
+ return counts, fmt.Errorf("failed to get space repo count: %w", err)
+ }
+
+ counts.Codes = codeCnt
+ counts.Datasets = dsCnt
+ counts.Models = modelCnt
+ counts.Spaces = spaceCnt
+ counts.TotalRepos = modelCnt + dsCnt + codeCnt + spaceCnt
+ return counts, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log/slog"
+ "math"
+ "strconv"
+ "strings"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type UserComponent interface {
+ Datasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error)
+ Models(ctx context.Context, req *types.UserModelsReq) ([]types.Model, int, error)
+ Codes(ctx context.Context, req *types.UserModelsReq) ([]types.Code, int, error)
+ Spaces(ctx context.Context, req *types.UserSpacesReq) ([]types.Space, int, error)
+ AddLikes(ctx context.Context, req *types.UserLikesRequest) error
+ // user likes collection
+ LikesCollection(ctx context.Context, req *types.UserSpacesReq) ([]types.Collection, int, error)
+ // UserCollections get collections of owner or visible to current user
+ Collections(ctx context.Context, req *types.UserCollectionReq) ([]types.Collection, int, error)
+ LikeCollection(ctx context.Context, req *types.UserLikesRequest) error
+ UnLikeCollection(ctx context.Context, req *types.UserLikesRequest) error
+ DeleteLikes(ctx context.Context, req *types.UserLikesRequest) error
+ LikesSpaces(ctx context.Context, req *types.UserSpacesReq) ([]types.Space, int, error)
+ LikesCodes(ctx context.Context, req *types.UserModelsReq) ([]types.Code, int, error)
+ LikesModels(ctx context.Context, req *types.UserModelsReq) ([]types.Model, int, error)
+ LikesDatasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error)
+ ListDeploys(ctx context.Context, repoType types.RepositoryType, req *types.DeployReq) ([]types.DeployRepo, int, error)
+ ListInstances(ctx context.Context, req *types.UserRepoReq) ([]types.DeployRepo, int, error)
+ ListServerless(ctx context.Context, req types.DeployReq) ([]types.DeployRepo, int, error)
+ CreateUserResource(ctx context.Context, req types.CreateUserResourceReq) error
+ DeleteUserResource(ctx context.Context, username string, orderDetailId int64) error
+ // GetUserResource
+ GetUserResource(ctx context.Context, req types.GetUserResourceReq) ([]types.UserResourcesResp, int, error)
+ GetUserByName(ctx context.Context, userName string) (*database.User, error)
+ Prompts(ctx context.Context, req *types.UserPromptsReq) ([]types.PromptRes, int, error)
+ Evaluations(ctx context.Context, req *types.UserEvaluationReq) ([]types.ArgoWorkFlowRes, int, error)
+}
+
+func NewUserComponent(config *config.Config) (UserComponent, error) {
+ c := &userComponentImpl{}
+ c.modelStore = database.NewModelStore()
+ c.userStore = database.NewUserStore()
+ c.datasetStore = database.NewDatasetStore()
+ c.codeStore = database.NewCodeStore()
+ c.spaceStore = database.NewSpaceStore()
+ c.namespaceStore = database.NewNamespaceStore()
+ c.collectionStore = database.NewCollectionStore()
+ c.spaceResourceStore = database.NewSpaceResourceStore()
+ c.userResourcesStore = database.NewUserResourcesStore()
+ var err error
+ c.spaceComponent, err = NewSpaceComponent(config)
+ if err != nil {
+ newError := fmt.Errorf("failed to create space component,error:%w", err)
+ return nil, newError
+ }
+ c.gitServer, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("failed to create git server,error:%w", err)
+ return nil, newError
+ }
+
+ c.repoComponent, err = NewRepoComponentImpl(config)
+ if err != nil {
+ newError := fmt.Errorf("failed to create repo component,error:%w", err)
+ return nil, newError
+ }
+ c.deployer = deploy.NewDeployer()
+ c.userLikeStore = database.NewUserLikesStore()
+ c.repoStore = database.NewRepoStore()
+ c.deployTaskStore = database.NewDeployTaskStore()
+ c.accountingComponent, err = NewAccountingComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ c.promptStore = database.NewPromptStore()
+ c.workflowStore = database.NewArgoWorkFlowStore()
+ return c, nil
+}
+
+type userComponentImpl struct {
+ userStore database.UserStore
+ modelStore database.ModelStore
+ datasetStore database.DatasetStore
+ codeStore database.CodeStore
+ spaceStore database.SpaceStore
+ namespaceStore database.NamespaceStore
+ gitServer gitserver.GitServer
+ spaceComponent SpaceComponent
+ repoComponent RepoComponent
+ deployer deploy.Deployer
+ userLikeStore database.UserLikesStore
+ repoStore database.RepoStore
+ deployTaskStore database.DeployTaskStore
+ collectionStore database.CollectionStore
+ accountingComponent AccountingComponent
+ spaceResourceStore database.SpaceResourceStore
+ userResourcesStore database.UserResourcesStore
+ promptStore database.PromptStore
+ workflowStore database.ArgoWorkFlowStore
+}
+
+func (c *userComponentImpl) Datasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error) {
+ var resDatasets []types.Dataset
+ userExists, err := c.userStore.IsExist(ctx, req.Owner)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !userExists {
+ return nil, 0, errors.New("user not exists")
+ }
+
+ if req.CurrentUser != "" {
+ cuserExists, err := c.userStore.IsExist(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of current user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !cuserExists {
+ return nil, 0, errors.New("current user not exists")
+ }
+ }
+
+ onlyPublic := req.Owner != req.CurrentUser
+ ds, total, err := c.datasetStore.ByUsername(ctx, req.Owner, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user datasets,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range ds {
+ resDatasets = append(resDatasets, types.Dataset{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resDatasets, total, nil
+}
+
+func (c *userComponentImpl) Models(ctx context.Context, req *types.UserModelsReq) ([]types.Model, int, error) {
+ var resModels []types.Model
+ userExists, err := c.userStore.IsExist(ctx, req.Owner)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !userExists {
+ return nil, 0, errors.New("user not exists")
+ }
+
+ if req.CurrentUser != "" {
+ cuserExists, err := c.userStore.IsExist(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of current user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !cuserExists {
+ return nil, 0, errors.New("current user not exists")
+ }
+ }
+
+ onlyPublic := req.Owner != req.CurrentUser
+ ms, total, err := c.modelStore.ByUsername(ctx, req.Owner, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user models,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range ms {
+ resModels = append(resModels, types.Model{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resModels, total, nil
+}
+
+func (c *userComponentImpl) Codes(ctx context.Context, req *types.UserModelsReq) ([]types.Code, int, error) {
+ var resCodes []types.Code
+ userExists, err := c.userStore.IsExist(ctx, req.Owner)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !userExists {
+ return nil, 0, errors.New("user not exists")
+ }
+
+ if req.CurrentUser != "" {
+ cuserExists, err := c.userStore.IsExist(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of current user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !cuserExists {
+ return nil, 0, errors.New("current user not exists")
+ }
+ }
+
+ onlyPublic := req.Owner != req.CurrentUser
+ ms, total, err := c.codeStore.ByUsername(ctx, req.Owner, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user codes,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range ms {
+ resCodes = append(resCodes, types.Code{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resCodes, total, nil
+}
+
+func (c *userComponentImpl) Spaces(ctx context.Context, req *types.UserSpacesReq) ([]types.Space, int, error) {
+ userExists, err := c.userStore.IsExist(ctx, req.Owner)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !userExists {
+ return nil, 0, errors.New("user not exists")
+ }
+
+ if req.CurrentUser != "" {
+ cuserExists, err := c.userStore.IsExist(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of current user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !cuserExists {
+ return nil, 0, errors.New("current user not exists")
+ }
+ }
+
+ return c.spaceComponent.UserSpaces(ctx, req)
+}
+
+func (c *userComponentImpl) AddLikes(ctx context.Context, req *types.UserLikesRequest) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return newError
+ }
+ var likesRepoIDs []int64
+ likesRepoIDs = append(likesRepoIDs, req.Repo_id)
+
+ var opts []database.SelectOption
+ opts = append(opts, database.Columns("id", "repository_type", "path", "user_id", "private"))
+
+ likesRepos, err := c.repoStore.FindByIds(ctx, likesRepoIDs, opts...)
+ if err != nil {
+ return fmt.Errorf("failed to get likes repositories by ids, error: %w", err)
+ }
+ likesRepos, err = c.repoComponent.VisiableToUser(ctx, likesRepos, req.CurrentUser)
+ if err != nil {
+ return fmt.Errorf("failed to check likes repositories visiable to user:%s, %w", req.CurrentUser, err)
+ }
+
+ if len(likesRepos) < 1 {
+ return fmt.Errorf("do not found likes repositories visiable to user:%s, %w", req.CurrentUser, err)
+ }
+
+ err = c.userLikeStore.Add(ctx, user.ID, req.Repo_id)
+ return err
+}
+
+// user likes collection
+func (c *userComponentImpl) LikesCollection(ctx context.Context, req *types.UserSpacesReq) ([]types.Collection, int, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return nil, 0, newError
+ }
+ collections, total, err := c.collectionStore.ByUserLikes(ctx, user.ID, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get collections by username,%w", err)
+ return nil, 0, newError
+ }
+
+ var newCollection []types.Collection
+ temporaryVariable, _ := json.Marshal(collections)
+ err = json.Unmarshal(temporaryVariable, &newCollection)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ return newCollection, total, nil
+}
+
+// UserCollections get collections of owner or visible to current user
+func (c *userComponentImpl) Collections(ctx context.Context, req *types.UserCollectionReq) ([]types.Collection, int, error) {
+ userExists, err := c.userStore.IsExist(ctx, req.Owner)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !userExists {
+ return nil, 0, errors.New("user not exists")
+ }
+
+ if req.CurrentUser != "" {
+ cuserExists, err := c.userStore.IsExist(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of current user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !cuserExists {
+ return nil, 0, errors.New("current user not exists")
+ }
+ }
+
+ onlyPublic := req.Owner != req.CurrentUser
+ collections, total, err := c.collectionStore.ByUsername(ctx, req.Owner, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get collections by username,%w", err)
+ return nil, 0, newError
+ }
+
+ var newCollection []types.Collection
+ temporaryVariable, _ := json.Marshal(collections)
+ err = json.Unmarshal(temporaryVariable, &newCollection)
+ if err != nil {
+ return nil, 0, err
+ }
+
+ return newCollection, total, nil
+}
+
+func (c *userComponentImpl) LikeCollection(ctx context.Context, req *types.UserLikesRequest) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return newError
+ }
+
+ collection, err := c.collectionStore.FindById(ctx, req.Collection_id)
+ if err != nil {
+ return fmt.Errorf("failed to get likes collection by id, error: %w", err)
+ }
+
+ if collection.Private && collection.UserID != user.ID {
+ return fmt.Errorf("no permission to like this collection for user:%s", req.CurrentUser)
+ }
+
+ err = c.userLikeStore.LikeCollection(ctx, user.ID, req.Collection_id)
+ return err
+}
+
+func (c *userComponentImpl) UnLikeCollection(ctx context.Context, req *types.UserLikesRequest) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ return newError
+ }
+ err = c.userLikeStore.UnLikeCollection(ctx, user.ID, req.Collection_id)
+ return err
+}
+
+func (c *userComponentImpl) DeleteLikes(ctx context.Context, req *types.UserLikesRequest) error {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ return newError
+ }
+ err = c.userLikeStore.Delete(ctx, user.ID, req.Repo_id)
+ return err
+}
+
+func (c *userComponentImpl) LikesSpaces(ctx context.Context, req *types.UserSpacesReq) ([]types.Space, int, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return nil, 0, newError
+ }
+ return c.spaceComponent.UserLikesSpaces(ctx, req, user.ID)
+}
+
+func (c *userComponentImpl) LikesCodes(ctx context.Context, req *types.UserModelsReq) ([]types.Code, int, error) {
+ var resCodes []types.Code
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return nil, 0, newError
+ }
+
+ ms, total, err := c.codeStore.UserLikesCodes(ctx, user.ID, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user codes,error:%w", err)
+ return nil, 0, newError
+ }
+
+ for _, data := range ms {
+ resCodes = append(resCodes, types.Code{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resCodes, total, nil
+}
+
+func (c *userComponentImpl) LikesModels(ctx context.Context, req *types.UserModelsReq) ([]types.Model, int, error) {
+ var resModels []types.Model
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return nil, 0, newError
+ }
+
+ ms, total, err := c.modelStore.UserLikesModels(ctx, user.ID, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user models,error:%w", err)
+ return nil, 0, newError
+ }
+
+ for _, data := range ms {
+ resModels = append(resModels, types.Model{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resModels, total, nil
+}
+
+func (c *userComponentImpl) LikesDatasets(ctx context.Context, req *types.UserDatasetsReq) ([]types.Dataset, int, error) {
+ var resDatasets []types.Dataset
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return nil, 0, newError
+ }
+
+ ds, total, err := c.datasetStore.UserLikesDatasets(ctx, user.ID, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user datasets,error:%w", err)
+ return nil, 0, newError
+ }
+
+ for _, data := range ds {
+ resDatasets = append(resDatasets, types.Dataset{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resDatasets, total, nil
+}
+
+func (c *userComponentImpl) ListDeploys(ctx context.Context, repoType types.RepositoryType, req *types.DeployReq) ([]types.DeployRepo, int, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user:%s, error:%w", req.CurrentUser, err)
+ return nil, 0, newError
+ }
+ deploys, total, err := c.deployTaskStore.ListDeployByUserID(ctx, user.ID, req)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user deploys for %s with error:%w", repoType, err)
+ return nil, 0, newError
+ }
+
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ })
+ if err != nil {
+ return nil, 0, fmt.Errorf("get price data with error: %w", err)
+ }
+
+ var resDeploys []types.DeployRepo
+ for _, deploy := range deploys {
+ d := &database.Deploy{
+ SvcName: deploy.SvcName,
+ ClusterID: deploy.ClusterID,
+ Status: deploy.Status,
+ }
+ endpoint, provider := c.repoComponent.GenerateEndpoint(ctx, d)
+ //get payMode
+ skuKind := types.SKUPayAsYouGo
+ if deploy.OrderDetailID != 0 {
+ skuKind = types.SKUTimeSpan
+ }
+ price := getAccountPrice(deploy.SKU, skuKind, priceData.Prices)
+ payMode := types.PayModeMinute
+ if price != nil {
+ payMode = types.PayMode(price.SkuUnitType)
+ }
+ repoPath := strings.TrimPrefix(deploy.GitPath, string(repoType)+"s_")
+ var hardware types.HardWare
+ _ = json.Unmarshal([]byte(deploy.Hardware), &hardware)
+ resourceType := ""
+ if hardware.Gpu.Num != "" {
+ resourceType = hardware.Gpu.Type
+ } else if hardware.Npu.Num != "" {
+ resourceType = hardware.Npu.Type
+ } else {
+ resourceType = hardware.Cpu.Type
+ }
+ tag := ""
+ tags, _ := c.repoStore.TagsWithCategory(ctx, deploy.RepoID, "task")
+ if len(tags) > 0 {
+ tag = tags[0].Name
+ }
+ resDeploys = append(resDeploys, types.DeployRepo{
+ DeployID: deploy.ID,
+ DeployName: deploy.DeployName,
+ Path: repoPath,
+ RepoID: deploy.RepoID,
+ SvcName: deploy.SvcName,
+ Status: deployStatusCodeToString(deploy.Status),
+ Hardware: deploy.Hardware,
+ Env: deploy.Env,
+ RuntimeFramework: deploy.RuntimeFramework,
+ ImageID: deploy.ImageID,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ GitPath: deploy.GitPath,
+ GitBranch: deploy.GitBranch,
+ ClusterID: deploy.ClusterID,
+ SecureLevel: deploy.SecureLevel,
+ CreatedAt: deploy.CreatedAt,
+ UpdatedAt: deploy.UpdatedAt,
+ Type: deploy.Type,
+ Endpoint: endpoint,
+ PayMode: payMode,
+ Provider: provider,
+ ResourceType: resourceType,
+ RepoTag: tag,
+ })
+ }
+ return resDeploys, total, nil
+}
+
+// get resource price
+func getAccountPrice(resourceId string, skuKind types.SKUKind, prices []database.AccountPrice) *database.AccountPrice {
+ for _, p := range prices {
+ if p.ResourceID == resourceId && p.SkuKind == skuKind {
+ return &p
+ }
+ }
+ return nil
+}
+
+func (c *userComponentImpl) ListInstances(ctx context.Context, req *types.UserRepoReq) ([]types.DeployRepo, int, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user:%s, error:%w", req.CurrentUser, err)
+ return nil, 0, newError
+ }
+ deploys, total, err := c.deployTaskStore.ListInstancesByUserID(ctx, user.ID, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user instances error:%w", err)
+ return nil, 0, newError
+ }
+
+ priceData, err := c.accountingComponent.QueryPricesBySKUType("", types.AcctPriceListReq{
+ SkuType: types.SKUCSGHub,
+ })
+ if err != nil {
+ return nil, 0, fmt.Errorf("get price data with error: %w", err)
+ }
+
+ var resDeploys []types.DeployRepo
+ for _, deploy := range deploys {
+ //get payMode
+ skuKind := types.SKUPayAsYouGo
+ if deploy.OrderDetailID != 0 {
+ skuKind = types.SKUTimeSpan
+ }
+ price := getAccountPrice(deploy.SKU, skuKind, priceData.Prices)
+ payMode := types.PayModeMinute
+ if price != nil {
+ payMode = types.PayMode(price.SkuUnitType)
+ }
+ repoPath := strings.TrimPrefix(deploy.GitPath, "models_")
+ resDeploys = append(resDeploys, types.DeployRepo{
+ DeployID: deploy.ID,
+ DeployName: deploy.DeployName,
+ Path: repoPath,
+ RepoID: deploy.RepoID,
+ SvcName: deploy.SvcName,
+ Status: deployStatusCodeToString(deploy.Status),
+ Hardware: deploy.Hardware,
+ Env: deploy.Env,
+ RuntimeFramework: deploy.RuntimeFramework,
+ ImageID: deploy.ImageID,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ GitPath: deploy.GitPath,
+ GitBranch: deploy.GitBranch,
+ ClusterID: deploy.ClusterID,
+ SecureLevel: deploy.SecureLevel,
+ CreatedAt: deploy.CreatedAt,
+ UpdatedAt: deploy.UpdatedAt,
+ Type: deploy.Type,
+ PayMode: payMode,
+ })
+ }
+ return resDeploys, total, nil
+}
+
+func (c *userComponentImpl) ListServerless(ctx context.Context, req types.DeployReq) ([]types.DeployRepo, int, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user:%s, error:%w", req.CurrentUser, err)
+ return nil, 0, newError
+ }
+ isAdmin := c.repoComponent.IsAdminRole(user)
+ if !isAdmin {
+ return nil, 0, fmt.Errorf("user %s does not have admin privileges", req.CurrentUser)
+ }
+ deploys, total, err := c.deployTaskStore.ListServerless(ctx, req)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user serverless for %s with error:%w", req.RepoType, err)
+ return nil, 0, newError
+ }
+
+ var resDeploys []types.DeployRepo
+ for _, deploy := range deploys {
+ repoPath := strings.TrimPrefix(deploy.GitPath, string(req.RepoType)+"s_")
+ resDeploys = append(resDeploys, types.DeployRepo{
+ DeployID: deploy.ID,
+ DeployName: deploy.DeployName,
+ Path: repoPath,
+ RepoID: deploy.RepoID,
+ SvcName: deploy.SvcName,
+ Status: deployStatusCodeToString(deploy.Status),
+ Hardware: deploy.Hardware,
+ Env: deploy.Env,
+ RuntimeFramework: deploy.RuntimeFramework,
+ ImageID: deploy.ImageID,
+ MinReplica: deploy.MinReplica,
+ MaxReplica: deploy.MaxReplica,
+ GitPath: deploy.GitPath,
+ GitBranch: deploy.GitBranch,
+ ClusterID: deploy.ClusterID,
+ SecureLevel: deploy.SecureLevel,
+ CreatedAt: deploy.CreatedAt,
+ UpdatedAt: deploy.UpdatedAt,
+ Type: deploy.Type,
+ SKU: deploy.SKU,
+ })
+ }
+ return resDeploys, total, nil
+}
+
+func (c *userComponentImpl) CreateUserResource(ctx context.Context, req types.CreateUserResourceReq) error {
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ return newError
+ }
+ var orderReq types.AcctOrderCreateReq
+ orderReq.OrderDetails = req.OrderDetails
+ orderReq.OrderUUID = uuid.New()
+ orderReq.UserUUID = user.UUID
+ urs, err := c.generateUserResource(ctx, req.Username, orderReq)
+ if err != nil {
+ return err
+ }
+ for _, ur := range urs {
+ err = c.userResourcesStore.AddUserResources(ctx, &ur)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (c *userComponentImpl) DeleteUserResource(ctx context.Context, username string, orderDetailId int64) error {
+ user, err := c.userStore.FindByUsername(ctx, username)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ return newError
+ }
+ err = c.userResourcesStore.DeleteUserResourcesByOrderDetailId(ctx, user.UUID, orderDetailId)
+ if err != nil {
+ return fmt.Errorf("failed to delete user resource, error:%w", err)
+ }
+ return nil
+}
+
+// generate user resource
+func (c *userComponentImpl) generateUserResource(ctx context.Context, username string, req types.AcctOrderCreateReq) ([]database.UserResources, error) {
+ //check resource available
+ for _, v := range req.OrderDetails {
+ resourceId, _ := strconv.ParseInt(v.ResourceID, 10, 64)
+ sp, err := c.spaceResourceStore.FindByID(ctx, resourceId)
+ if err != nil {
+ return nil, err
+ }
+ var hardware types.HardWare
+ err = json.Unmarshal([]byte(sp.Resources), &hardware)
+ if err != nil {
+ return nil, err
+ }
+ _, err = c.deployer.CheckResourceAvailable(ctx, sp.ClusterID, 0, &hardware)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create order, %w", err)
+ }
+ }
+ // to do get order details
+ order, err := c.accountingComponent.CreateOrder(username, req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create order for user %s, order id: %s, error:%w", username, req.OrderUUID, err)
+ }
+ var urs []database.UserResources
+ for _, v := range order.Details {
+ resourceId, _ := strconv.ParseInt(v.ResourceID, 10, 64)
+ sp, err := c.spaceResourceStore.FindByID(ctx, resourceId)
+ if err != nil {
+ return nil, err
+ }
+ var hardware types.HardWare
+ err = json.Unmarshal([]byte(sp.Resources), &hardware)
+ if err != nil {
+ return nil, err
+ }
+ var xpu_num = 0
+ if hardware.Gpu.Num != "" {
+ xpu_num, err = strconv.Atoi(hardware.Gpu.Num)
+ } else if hardware.Npu.Num != "" {
+ xpu_num, err = strconv.Atoi(hardware.Npu.Num)
+ }
+ if err != nil {
+ return nil, err
+ }
+ _, err = c.deployer.CheckResourceAvailable(ctx, sp.ClusterID, 0, &hardware)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check resource, %w", err)
+ }
+ var ur = database.UserResources{}
+ ur.UserUID = order.UserUUID
+ ur.OrderId = order.OrderUUID
+ ur.OrderDetailId = v.ID
+ ur.ResourceId = resourceId
+ ur.StartTime = v.BeginTime
+ ur.EndTime = v.EndTime
+ ur.XPUNum = xpu_num
+ ur.PayMode = v.SkuUnitType
+ ur.Price = math.Abs(v.Amount)
+ urs = append(urs, ur)
+ }
+ return urs, nil
+}
+
+// GetUserResource
+func (c *userComponentImpl) GetUserResource(ctx context.Context, req types.GetUserResourceReq) ([]types.UserResourcesResp, int, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ return nil, 0, newError
+ }
+ var urs []types.UserResourcesResp
+ data, total, err := c.userResourcesStore.GetUserResourcesByUserUID(ctx, req.PageSize, req.Page, user.UUID)
+ if err != nil {
+ return nil, 0, err
+ }
+ for _, v := range data {
+ deployName := "-"
+ repoPath := ""
+ deployType := types.UnknownType
+ if v.Deploy != nil {
+ deployName = v.Deploy.DeployName
+ repoPath = strings.TrimPrefix(v.Deploy.GitPath, "models_")
+ repoPath = strings.TrimPrefix(repoPath, "spaces_")
+ deployType = v.Deploy.Type
+ }
+ resource := ""
+ resourceType := ""
+ if v.SpaceResource != nil {
+ var hardware types.HardWare
+ _ = json.Unmarshal([]byte(v.SpaceResource.Resources), &hardware)
+ resource = hardware.Cpu.Num + "vCPU·" + hardware.Memory
+ if hardware.Gpu.Num != "" {
+ resourceType = hardware.Gpu.Type
+ resource += "·" + hardware.Gpu.Num + "GPU"
+ } else if hardware.Npu.Num != "" {
+ resourceType = hardware.Npu.Type
+ resource += "·" + hardware.Npu.Num + "NPU"
+ } else {
+ resourceType = hardware.Cpu.Type
+ }
+ }
+ urs = append(urs, types.UserResourcesResp{
+ ID: v.ID,
+ OrderId: v.OrderId,
+ OrderDetailId: v.OrderDetailId,
+ ResourceId: v.ResourceId,
+ CreatedAt: v.CreatedAt,
+ StartTime: v.StartTime,
+ EndTime: v.EndTime,
+ XPUNum: v.XPUNum,
+ PayMode: v.PayMode,
+ Price: v.Price,
+ DeployName: deployName,
+ DeployID: v.DeployId,
+ RepoPath: repoPath,
+ Resource: resource,
+ ResourceType: resourceType,
+ DeployType: deployType,
+ })
+ }
+ return urs, total, nil
+}
+
+func (c *userComponentImpl) GetUserByName(ctx context.Context, userName string) (*database.User, error) {
+ user, err := c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check for the presence of the user %s,error:%w", userName, err)
+ }
+ return &user, nil
+}
+
+func (c *userComponentImpl) Prompts(ctx context.Context, req *types.UserPromptsReq) ([]types.PromptRes, int, error) {
+ var resPrompts []types.PromptRes
+ userExists, err := c.userStore.IsExist(ctx, req.Owner)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !userExists {
+ return nil, 0, errors.New("user not exists")
+ }
+
+ if req.CurrentUser != "" {
+ cuserExists, err := c.userStore.IsExist(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of current user,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ if !cuserExists {
+ return nil, 0, errors.New("current user not exists")
+ }
+ }
+
+ onlyPublic := req.Owner != req.CurrentUser
+ ds, total, err := c.promptStore.ByUsername(ctx, req.Owner, req.PageSize, req.Page, onlyPublic)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user prompts,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+
+ for _, data := range ds {
+
+ resPrompts = append(resPrompts, types.PromptRes{
+ ID: data.ID,
+ Name: data.Repository.Name,
+ Nickname: data.Repository.Nickname,
+ Description: data.Repository.Description,
+ Likes: data.Repository.Likes,
+ Downloads: data.Repository.DownloadCount,
+ Path: data.Repository.Path,
+ RepositoryID: data.RepositoryID,
+ Private: data.Repository.Private,
+ CreatedAt: data.CreatedAt,
+ UpdatedAt: data.Repository.UpdatedAt,
+ })
+ }
+
+ return resPrompts, total, nil
+}
+
+func (c *userComponentImpl) Evaluations(ctx context.Context, req *types.UserEvaluationReq) ([]types.ArgoWorkFlowRes, int, error) {
+
+ _, err := c.userStore.FindByUsername(ctx, req.CurrentUser)
+ if err != nil {
+ newError := fmt.Errorf("failed to check for the presence of the user, error:%w", err)
+ return nil, 0, newError
+ }
+
+ res, err := c.deployer.ListEvaluations(ctx, req.CurrentUser, req.PageSize, req.Page)
+ if err != nil {
+ newError := fmt.Errorf("failed to get user evaluations,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, 0, newError
+ }
+ return res.List, res.Total, nil
+}
+
+
+
package component
+
+import (
+ "github.com/google/wire"
+ mock_accounting "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/accounting"
+ mock_deploy "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/deploy"
+ mock_git "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/git/gitserver"
+ mock_mirror "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/git/mirrorserver"
+ mock_preader "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/parquet"
+ mock_rpc "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/rpc"
+ mock_s3 "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/builder/store/s3"
+ mock_component "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/component"
+ mock_cache "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/mirror/cache"
+ mock_mirror_queue "opencsg.com/csghub-server/_mocks/opencsg.com/csghub-server/mirror/queue"
+ "opencsg.com/csghub-server/builder/accounting"
+ "opencsg.com/csghub-server/builder/deploy"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/mirrorserver"
+ "opencsg.com/csghub-server/builder/llm"
+ "opencsg.com/csghub-server/builder/parquet"
+ "opencsg.com/csghub-server/builder/rpc"
+ "opencsg.com/csghub-server/builder/store/s3"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/tests"
+ "opencsg.com/csghub-server/mirror/cache"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+type mockedComponents struct {
+ accounting *mock_component.MockAccountingComponent
+ repo *mock_component.MockRepoComponent
+ tag *mock_component.MockTagComponent
+ space *mock_component.MockSpaceComponent
+ runtimeArchitecture *mock_component.MockRuntimeArchitectureComponent
+ sensitive *mock_component.MockSensitiveComponent
+}
+
+var MockedStoreSet = wire.NewSet(
+ tests.NewMockStores,
+)
+
+var MockedComponentSet = wire.NewSet(
+ mock_component.NewMockAccountingComponent,
+ wire.Bind(new(AccountingComponent), new(*mock_component.MockAccountingComponent)),
+ mock_component.NewMockRepoComponent,
+ wire.Bind(new(RepoComponent), new(*mock_component.MockRepoComponent)),
+ mock_component.NewMockTagComponent,
+ wire.Bind(new(TagComponent), new(*mock_component.MockTagComponent)),
+ mock_component.NewMockSpaceComponent,
+ wire.Bind(new(SpaceComponent), new(*mock_component.MockSpaceComponent)),
+ mock_component.NewMockRuntimeArchitectureComponent,
+ wire.Bind(new(RuntimeArchitectureComponent), new(*mock_component.MockRuntimeArchitectureComponent)),
+ mock_component.NewMockSensitiveComponent,
+ wire.Bind(new(SensitiveComponent), new(*mock_component.MockSensitiveComponent)),
+)
+
+var MockedGitServerSet = wire.NewSet(
+ mock_git.NewMockGitServer,
+ wire.Bind(new(gitserver.GitServer), new(*mock_git.MockGitServer)),
+)
+
+var MockedUserSvcSet = wire.NewSet(
+ mock_rpc.NewMockUserSvcClient,
+ wire.Bind(new(rpc.UserSvcClient), new(*mock_rpc.MockUserSvcClient)),
+)
+
+var MockedS3Set = wire.NewSet(
+ mock_s3.NewMockClient,
+ wire.Bind(new(s3.Client), new(*mock_s3.MockClient)),
+)
+
+var MockedDeployerSet = wire.NewSet(
+ mock_deploy.NewMockDeployer,
+ wire.Bind(new(deploy.Deployer), new(*mock_deploy.MockDeployer)),
+)
+
+var MockedCacheSet = wire.NewSet(
+ mock_cache.NewMockCache,
+ wire.Bind(new(cache.Cache), new(*mock_cache.MockCache)),
+)
+
+var MockedMirrorServerSet = wire.NewSet(
+ mock_mirror.NewMockMirrorServer,
+ wire.Bind(new(mirrorserver.MirrorServer), new(*mock_mirror.MockMirrorServer)),
+)
+
+var MockedMirrorQueueSet = wire.NewSet(
+ mock_mirror_queue.NewMockPriorityQueue,
+ wire.Bind(new(queue.PriorityQueue), new(*mock_mirror_queue.MockPriorityQueue)),
+)
+
+var MockedAccountingClientSet = wire.NewSet(
+ mock_accounting.NewMockAccountingClient,
+ wire.Bind(new(accounting.AccountingClient), new(*mock_accounting.MockAccountingClient)),
+)
+
+var MockedParquetReaderSet = wire.NewSet(
+ mock_preader.NewMockReader,
+ wire.Bind(new(parquet.Reader), new(*mock_preader.MockReader)),
+)
+
+var MockedModerationSvcClientSet = wire.NewSet(
+ mock_rpc.NewMockModerationSvcClient,
+ wire.Bind(new(rpc.ModerationSvcClient), new(*mock_rpc.MockModerationSvcClient)),
+)
+
+type Mocks struct {
+ stores *tests.MockStores
+ components *mockedComponents
+ gitServer *mock_git.MockGitServer
+ userSvcClient *mock_rpc.MockUserSvcClient
+ s3Client *mock_s3.MockClient
+ mirrorServer *mock_mirror.MockMirrorServer
+ mirrorQueue *mock_mirror_queue.MockPriorityQueue
+ deployer *mock_deploy.MockDeployer
+ cache *mock_cache.MockCache
+ accountingClient *mock_accounting.MockAccountingClient
+ preader *mock_preader.MockReader
+ moderationClient *mock_rpc.MockModerationSvcClient
+}
+
+var AllMockSet = wire.NewSet(
+ wire.Struct(new(mockedComponents), "*"),
+ wire.Struct(new(Mocks), "*"),
+)
+
+func ProvideTestConfig() *config.Config {
+ return &config.Config{}
+}
+
+var MockSuperSet = wire.NewSet(
+ MockedComponentSet, AllMockSet, MockedStoreSet, MockedGitServerSet, MockedUserSvcSet,
+ MockedS3Set, MockedDeployerSet, MockedCacheSet, ProvideTestConfig, MockedMirrorServerSet,
+ MockedMirrorQueueSet, MockedAccountingClientSet, MockedParquetReaderSet,
+ MockedModerationSvcClientSet,
+)
+
+func NewTestRepoComponent(config *config.Config, stores *tests.MockStores, rpcUser rpc.UserSvcClient, gitServer gitserver.GitServer, tagComponent TagComponent, s3Client s3.Client, deployer deploy.Deployer, cache cache.Cache, accountingComponent AccountingComponent, mq queue.PriorityQueue, mirrorServer mirrorserver.MirrorServer) *repoComponentImpl {
+ return &repoComponentImpl{
+ userStore: stores.User,
+ repoStore: stores.Repo,
+ repoRelationsStore: stores.RepoRelation,
+ namespaceStore: stores.Namespace,
+ userSvcClient: rpcUser,
+ config: config,
+ git: gitServer,
+ tagComponent: tagComponent,
+ s3Client: s3Client,
+ lfsMetaObjectStore: stores.LfsMetaObject,
+ mirrorStore: stores.Mirror,
+ mirrorSourceStore: stores.MirrorSource,
+ tokenStore: stores.AccessToken,
+ syncVersionStore: stores.SyncVersion,
+ syncClientSettingStore: stores.SyncClientSetting,
+ runtimeFrameworksStore: stores.RuntimeFramework,
+ deployTaskStore: stores.DeployTask,
+ deployer: deployer,
+ userResourcesStore: stores.UserResources,
+ clusterInfoStore: stores.ClusterInfo,
+ syncCache: cache,
+ accountingComponent: accountingComponent,
+ spaceResourceStore: stores.SpaceResource,
+ mq: mq,
+ mirrorServer: mirrorServer,
+ }
+}
+
+var RepoComponentSet = wire.NewSet(NewTestRepoComponent)
+
+func NewTestPromptComponent(config *config.Config, stores *tests.MockStores, repoComponent RepoComponent, rpcUser rpc.UserSvcClient, gitServer gitserver.GitServer) *promptComponentImpl {
+ config.APIServer.PublicDomain = "https://foo.com"
+ config.APIServer.SSHDomain = "ssh://test@127.0.0.1"
+ return &promptComponentImpl{
+ userStore: stores.User,
+ userLikeStore: stores.UserLikes,
+ promptConvStore: stores.PromptConversation,
+ promptPrefixStore: stores.PromptPrefix,
+ llmConfigStore: stores.LLMConfig,
+ promptStore: stores.Prompt,
+ namespaceStore: stores.Namespace,
+ userSvcClient: rpcUser,
+ gitServer: gitServer,
+ repoStore: stores.Repo,
+ llmClient: llm.NewClient(),
+ config: config,
+ repoComponent: repoComponent,
+ }
+}
+
+var PromptComponentSet = wire.NewSet(NewTestPromptComponent)
+
+func NewTestUserComponent(
+ stores *tests.MockStores,
+ gitServer gitserver.GitServer,
+ spaceComponent SpaceComponent,
+ repoComponent RepoComponent,
+ deployer deploy.Deployer,
+ accountingComponent AccountingComponent,
+) *userComponentImpl {
+ return &userComponentImpl{
+ userStore: stores.User,
+ modelStore: stores.Model,
+ datasetStore: stores.Dataset,
+ codeStore: stores.Code,
+ spaceStore: stores.Space,
+ namespaceStore: stores.Namespace,
+ gitServer: gitServer,
+ spaceComponent: spaceComponent,
+ repoComponent: repoComponent,
+ deployer: deployer,
+ userLikeStore: stores.UserLikes,
+ repoStore: stores.Repo,
+ deployTaskStore: stores.DeployTask,
+ collectionStore: stores.Collection,
+ accountingComponent: accountingComponent,
+ spaceResourceStore: stores.SpaceResource,
+ userResourcesStore: stores.UserResources,
+ promptStore: stores.Prompt,
+ workflowStore: stores.Workflow,
+ }
+}
+
+var UserComponentSet = wire.NewSet(NewTestUserComponent)
+
+func NewTestSpaceComponent(
+ stores *tests.MockStores,
+ repoComponent RepoComponent,
+ git gitserver.GitServer,
+ deployer deploy.Deployer,
+ accountingComponent AccountingComponent,
+ config *config.Config,
+ userSvcClient rpc.UserSvcClient,
+) *spaceComponentImpl {
+ return &spaceComponentImpl{
+ repoComponent: repoComponent,
+ git: git,
+ spaceStore: stores.Space,
+ spaceSdkStore: stores.SpaceSdk,
+ spaceResourceStore: stores.SpaceResource,
+ repoStore: stores.Repo,
+ userStore: stores.User,
+ deployer: deployer,
+ publicRootDomain: config.Space.PublicRootDomain,
+ accountingComponent: accountingComponent,
+ userResourcesStore: stores.UserResources,
+ serverBaseUrl: config.APIServer.PublicDomain,
+ userLikesStore: stores.UserLikes,
+ config: config,
+ userSvcClient: userSvcClient,
+ deployTaskStore: stores.DeployTask,
+ }
+}
+
+var SpaceComponentSet = wire.NewSet(NewTestSpaceComponent)
+
+func NewTestModelComponent(
+ config *config.Config,
+ stores *tests.MockStores,
+ repoComponent RepoComponent,
+ spaceComponent SpaceComponent,
+ deployer deploy.Deployer,
+ accountingComponent AccountingComponent,
+ runtimeArchComponent RuntimeArchitectureComponent,
+ gitServer gitserver.GitServer,
+ userSvcClient rpc.UserSvcClient,
+) *modelComponentImpl {
+ config.APIServer.PublicDomain = "https://foo.com"
+ config.APIServer.SSHDomain = "ssh://test@127.0.0.1"
+ return &modelComponentImpl{
+ config: config,
+ repoComponent: repoComponent,
+ spaceComponent: spaceComponent,
+ modelStore: stores.Model,
+ repoStore: stores.Repo,
+ spaceResourceStore: stores.SpaceResource,
+ userStore: stores.User,
+ deployer: deployer,
+ accountingComponent: accountingComponent,
+ tagStore: stores.Tag,
+ runtimeArchComponent: runtimeArchComponent,
+ datasetStore: stores.Dataset,
+ recomStore: stores.Recom,
+ gitServer: gitServer,
+ userLikesStore: stores.UserLikes,
+ repoRuntimeFrameworkStore: stores.RepoRuntimeFramework,
+ deployTaskStore: stores.DeployTask,
+ runtimeFrameworksStore: stores.RuntimeFramework,
+ userSvcClient: userSvcClient,
+ }
+}
+
+var ModelComponentSet = wire.NewSet(NewTestModelComponent)
+
+func NewTestAccountingComponent(stores *tests.MockStores, accountingClient accounting.AccountingClient) *accountingComponentImpl {
+ return &accountingComponentImpl{
+ acctountingClient: accountingClient,
+ userStore: stores.User,
+ deployTaskStore: stores.DeployTask,
+ }
+}
+
+var AccountingComponentSet = wire.NewSet(NewTestAccountingComponent)
+
+func NewTestDatasetViewerComponent(stores *tests.MockStores, cfg *config.Config, repoComponent RepoComponent, gitServer gitserver.GitServer, preader parquet.Reader) *datasetViewerComponentImpl {
+ return &datasetViewerComponentImpl{
+ cfg: cfg,
+ repoStore: stores.Repo,
+ repoComponent: repoComponent,
+ gitServer: gitServer,
+ preader: preader,
+ }
+}
+
+var DatasetViewerComponentSet = wire.NewSet(NewTestDatasetViewerComponent)
+
+func NewTestGitHTTPComponent(
+ config *config.Config,
+ stores *tests.MockStores,
+ repoComponent RepoComponent,
+ gitServer gitserver.GitServer,
+ s3Client s3.Client,
+) *gitHTTPComponentImpl {
+ config.APIServer.PublicDomain = "https://foo.com"
+ config.APIServer.SSHDomain = "ssh://test@127.0.0.1"
+ return &gitHTTPComponentImpl{
+ config: config,
+ repoComponent: repoComponent,
+ repoStore: stores.Repo,
+ userStore: stores.User,
+ gitServer: gitServer,
+ s3Client: s3Client,
+ lfsMetaObjectStore: stores.LfsMetaObject,
+ lfsLockStore: stores.LfsLock,
+ }
+}
+
+var GitHTTPComponentSet = wire.NewSet(NewTestGitHTTPComponent)
+
+func NewTestDiscussionComponent(
+ stores *tests.MockStores,
+) *discussionComponentImpl {
+ return &discussionComponentImpl{
+ repoStore: stores.Repo,
+ userStore: stores.User,
+ discussionStore: stores.Discussion,
+ }
+}
+
+var DiscussionComponentSet = wire.NewSet(NewTestDiscussionComponent)
+
+func NewTestRuntimeArchitectureComponent(stores *tests.MockStores, repoComponent RepoComponent, gitServer gitserver.GitServer) *runtimeArchitectureComponentImpl {
+ return &runtimeArchitectureComponentImpl{
+ repoComponent: repoComponent,
+ repoStore: stores.Repo,
+ repoRuntimeFrameworkStore: stores.RepoRuntimeFramework,
+ runtimeFrameworksStore: stores.RuntimeFramework,
+ runtimeArchStore: stores.RuntimeArch,
+ resouceModelStore: stores.ResourceModel,
+ tagStore: stores.Tag,
+ gitServer: gitServer,
+ }
+}
+
+var RuntimeArchComponentSet = wire.NewSet(NewTestRuntimeArchitectureComponent)
+
+func NewTestMirrorComponent(config *config.Config, stores *tests.MockStores, mirrorServer mirrorserver.MirrorServer, repoComponent RepoComponent, gitServer gitserver.GitServer, s3Client s3.Client, mq queue.PriorityQueue) *mirrorComponentImpl {
+ return &mirrorComponentImpl{
+ tokenStore: stores.GitServerAccessToken,
+ mirrorServer: mirrorServer,
+ repoComp: repoComponent,
+ git: gitServer,
+ s3Client: s3Client,
+ modelStore: stores.Model,
+ datasetStore: stores.Dataset,
+ codeStore: stores.Code,
+ repoStore: stores.Repo,
+ mirrorStore: stores.Mirror,
+ mirrorSourceStore: stores.MirrorSource,
+ namespaceStore: stores.Namespace,
+ userStore: stores.User,
+ config: config,
+ mq: mq,
+ }
+}
+
+var MirrorComponentSet = wire.NewSet(NewTestMirrorComponent)
+
+func NewTestCollectionComponent(stores *tests.MockStores, userSvcClient rpc.UserSvcClient, spaceComponent SpaceComponent) *collectionComponentImpl {
+ return &collectionComponentImpl{
+ collectionStore: stores.Collection,
+ orgStore: stores.Org,
+ repoStore: stores.Repo,
+ userStore: stores.User,
+ userLikesStore: stores.UserLikes,
+ userSvcClient: userSvcClient,
+ spaceComponent: spaceComponent,
+ }
+}
+
+var CollectionComponentSet = wire.NewSet(NewTestCollectionComponent)
+
+func NewTestDatasetComponent(config *config.Config, stores *tests.MockStores, repoComponent RepoComponent, userSvcClient rpc.UserSvcClient, sensitiveComponent SensitiveComponent, gitServer gitserver.GitServer) *datasetComponentImpl {
+ return &datasetComponentImpl{
+ config: config,
+ repoComponent: repoComponent,
+ tagStore: stores.Tag,
+ datasetStore: stores.Dataset,
+ repoStore: stores.Repo,
+ namespaceStore: stores.Namespace,
+ userStore: stores.User,
+ sensitiveComponent: sensitiveComponent,
+ gitServer: gitServer,
+ userLikesStore: stores.UserLikes,
+ userSvcClient: userSvcClient,
+ }
+}
+
+var DatasetComponentSet = wire.NewSet(NewTestDatasetComponent)
+
+func NewTestCodeComponent(config *config.Config, stores *tests.MockStores, repoComponent RepoComponent, userSvcClient rpc.UserSvcClient, gitServer gitserver.GitServer) *codeComponentImpl {
+ return &codeComponentImpl{
+ config: config,
+ repoComponent: repoComponent,
+ codeStore: stores.Code,
+ repoStore: stores.Repo,
+ userLikesStore: stores.UserLikes,
+ gitServer: gitServer,
+ userSvcClient: userSvcClient,
+ }
+}
+
+var CodeComponentSet = wire.NewSet(NewTestCodeComponent)
+
+func NewTestMultiSyncComponent(config *config.Config, stores *tests.MockStores, gitServer gitserver.GitServer) *multiSyncComponentImpl {
+ return &multiSyncComponentImpl{
+ multiSyncStore: stores.MultiSync,
+ repoStore: stores.Repo,
+ modelStore: stores.Model,
+ datasetStore: stores.Dataset,
+ namespaceStore: stores.Namespace,
+ userStore: stores.User,
+ syncVersionStore: stores.SyncVersion,
+ tagStore: stores.Tag,
+ fileStore: stores.File,
+ gitServer: gitServer,
+ }
+}
+
+var MultiSyncComponentSet = wire.NewSet(NewTestMultiSyncComponent)
+
+func NewTestInternalComponent(config *config.Config, stores *tests.MockStores, repoComponent RepoComponent, gitServer gitserver.GitServer) *internalComponentImpl {
+ return &internalComponentImpl{
+ config: config,
+ sshKeyStore: stores.SSH,
+ repoStore: stores.Repo,
+ tokenStore: stores.AccessToken,
+ namespaceStore: stores.Namespace,
+ repoComponent: repoComponent,
+ gitServer: gitServer,
+ }
+}
+
+var InternalComponentSet = wire.NewSet(NewTestInternalComponent)
+
+func NewTestMirrorSourceComponent(config *config.Config, stores *tests.MockStores) *mirrorSourceComponentImpl {
+ return &mirrorSourceComponentImpl{
+ mirrorSourceStore: stores.MirrorSource,
+ userStore: stores.User,
+ }
+}
+
+var MirrorSourceComponentSet = wire.NewSet(NewTestMirrorSourceComponent)
+
+func NewTestSpaceResourceComponent(config *config.Config, stores *tests.MockStores, deployer deploy.Deployer, accountComponent AccountingComponent) *spaceResourceComponentImpl {
+ return &spaceResourceComponentImpl{
+ spaceResourceStore: stores.SpaceResource,
+ deployer: deployer,
+ userResourceStore: stores.UserResources,
+ userStore: stores.User,
+ accountComponent: accountComponent,
+ }
+}
+
+var SpaceResourceComponentSet = wire.NewSet(NewTestSpaceResourceComponent)
+
+func NewTestTagComponent(config *config.Config, stores *tests.MockStores, sensitiveChecker rpc.ModerationSvcClient) *tagComponentImpl {
+ return &tagComponentImpl{
+ tagStore: stores.Tag,
+ repoStore: stores.Repo,
+ sensitiveChecker: sensitiveChecker,
+ }
+}
+
+var TagComponentSet = wire.NewSet(NewTestTagComponent)
+
+func NewTestRecomComponent(config *config.Config, stores *tests.MockStores, gitServer gitserver.GitServer) *recomComponentImpl {
+ return &recomComponentImpl{
+ recomStore: stores.Recom,
+ userStore: stores.User,
+ repoStore: stores.Repo,
+ gitServer: gitServer,
+ }
+}
+
+var RecomComponentSet = wire.NewSet(NewTestRecomComponent)
+
+func NewTestSpaceSdkComponent(config *config.Config, stores *tests.MockStores) *spaceSdkComponentImpl {
+ return &spaceSdkComponentImpl{
+ spaceSdkStore: stores.SpaceSdk,
+ }
+}
+
+var SpaceSdkComponentSet = wire.NewSet(NewTestSpaceSdkComponent)
+
+
+
// Package docs Code generated by swaggo/swag. DO NOT EDIT
+package docs
+
+import "github.com/swaggo/swag"
+
+const docTemplate = `{
+ "schemes": {{ marshal .Schemes }},
+ "swagger": "2.0",
+ "info": {
+ "description": "{{escape .Description}}",
+ "title": "{{.Title}}",
+ "contact": {},
+ "version": "{{.Version}}"
+ },
+ "host": "{{.Host}}",
+ "basePath": "{{.BasePath}}",
+ "paths": {
+ "/accounting/credit/balance": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get all users balance",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Get all users balance",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/credit/{id}/balance": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get user balance by user uuid",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Get user balance by user uuid",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "user uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/credit/{id}/bills": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List user bills by user uuid and start date and end date",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "List user bills by user uuid and start date and end date",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "user uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 10,
+ 11,
+ 12,
+ 20
+ ],
+ "type": "integer",
+ "default": 10,
+ "description": "scene",
+ "name": "scene",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "start_date, format: '2024-06-12'",
+ "name": "start_date",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "end_date, format: '2024-07-12'",
+ "name": "end_date",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/credit/{id}/recharge": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Recharge fee for account",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Recharge fee for account",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "user uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RECHARGE_REQ"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/credit/{id}/statements": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List statements by user uuid and start time and end time",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "List statements by user uuid and start time and end time",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "user uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 10,
+ 11,
+ 12,
+ 20
+ ],
+ "type": "integer",
+ "default": 10,
+ "description": "scene",
+ "name": "scene",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "instance name",
+ "name": "instance_name",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "start_time, format: '2024-06-12 08:27:22'",
+ "name": "start_time",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "end_time, format: '2024-06-12 17:17:22'",
+ "name": "end_time",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/metering/{id}/statements": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List meterings by user uuid and start time and end time",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "List meterings by user uuid and start time and end time",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "user uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 10,
+ 11,
+ 12,
+ 20
+ ],
+ "type": "integer",
+ "default": 10,
+ "description": "scene",
+ "name": "scene",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "instance name",
+ "name": "instance_name",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "start_time, format: '2024-06-12 08:27:22'",
+ "name": "start_time",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "end_time, format: '2024-06-12 17:17:22'",
+ "name": "end_time",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/multisync/download": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get account quota statement",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Get account quota statement",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "repo path",
+ "name": "repo_path",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo type",
+ "name": "repo_type",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/multisync/downloads": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Add download count",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Add download count",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ACCT_QUOTA_STATEMENT_REQ"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/multisync/quota": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get account quota by user id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Get account quota by user id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/multisync/quotas": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Add or update account quota",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Add or update account quota",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ACCT_QUOTA_REQ"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/price": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List sku prices",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "List sku prices",
+ "parameters": [
+ {
+ "enum": [
+ "1",
+ "2"
+ ],
+ "type": "string",
+ "default": "1",
+ "description": "sku_type",
+ "name": "sku_type",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "sku_kind",
+ "name": "sku_kind",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "resource_id",
+ "name": "resource_id",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Add sku price",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Add sku price",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.AcctPriceCreateReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/price/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get price by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Get price by id",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Update sku price",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Update sku price",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.AcctPriceCreateReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Delete price by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Delete price by id",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/recharge/create-pay-order": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Create recharge order",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Create recharge order",
+ "parameters": [
+ {
+ "description": "Recharge request payload",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.AcctRechargeReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.AcctRechargeResp"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/recharge/user-recharge-list": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List current user recharge list by start_time and end_time and query",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "List current user recharge list by start_time and end_time and query",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Recharge status, options: 'succeed, waitpay'",
+ "name": "recharge_status",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "Recharge payment type, options: 'wx_pub_qr, alipay_qr'",
+ "name": "recharge_payment_type",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "Query string against recharge uuid",
+ "name": "query",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "Start time, format: '2024-06-12 08:27:22'",
+ "name": "start_time",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "End time, format: '2024-06-12 17:17:22'",
+ "name": "end_time",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "Current user identifier",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "Items per page",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "Page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/accounting/recharge/{id}/status": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Fetch recharge order status by recharge id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Accounting"
+ ],
+ "summary": "Fetch recharge order status by recharge id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "recharge uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.RechargeStatusResp"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/cluster": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get cluster list",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Cluster"
+ ],
+ "summary": "Get cluster list",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/cluster/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get cluster by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Cluster"
+ ],
+ "summary": "Get cluster by id",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/codes": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get visiable codes for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Code"
+ ],
+ "summary": "Get Visiable codes for current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by task tag",
+ "name": "task_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by framework tag",
+ "name": "framework_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by license tag",
+ "name": "license_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by language tag",
+ "name": "language_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "opencsg",
+ "huggingface",
+ "local"
+ ],
+ "type": "string",
+ "description": "source",
+ "name": "source",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Code"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new code",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Code"
+ ],
+ "summary": "Create a new code",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateCodeReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Code"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/codes/{namespace}/{name}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get code detail",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Code"
+ ],
+ "summary": "Get code detail",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Code"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exists code",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Code"
+ ],
+ "summary": "Update a exists code",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateCodeReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Code"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exists code",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Code"
+ ],
+ "summary": "Delete a exists code",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/codes/{namespace}/{name}/relations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Code"
+ ],
+ "summary": "Get code related assets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Relations"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/collections": {
+ "get": {
+ "description": "get all collections",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "get all collections",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "default": "\"trending\"",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "description": "create a collection",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "create a collection",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateCollectionReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/collections/{id}": {
+ "get": {
+ "description": "get a collection detail",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "get a collection detail",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "description": "update a collection",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "update a collection",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateCollectionReq"
+ }
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "description": "delete a exists collection",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "Delete a exists collection",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/collections/{id}/repos": {
+ "post": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "description": "Add repos to a collection",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "Add repos to a collection",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateCollectionReposReq"
+ }
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Collection"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "description": "remove repos from a collection",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Collection"
+ ],
+ "summary": "remove repos from a collection",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateCollectionReposReq"
+ }
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/comments/{id}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a comment content by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Update a comment content by id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the comment id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner of the comment",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/component.UpdateCommentRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a comment by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Delete a comment by id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the comment id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner of the comment",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get visiable datasets for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get Visiable datasets for current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by task tag",
+ "name": "task_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by framework tag",
+ "name": "framework_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by license tag",
+ "name": "license_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by language tag",
+ "name": "language_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "opencsg",
+ "huggingface",
+ "local"
+ ],
+ "type": "string",
+ "description": "source",
+ "name": "source",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new dataset",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Create a new dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateDatasetReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets/{namespace}/{name}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get dataset detail",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get dataset detail",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exists dataset",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Update a exists dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateDatasetReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Dataset"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exists dataset",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Delete a exists dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets/{namespace}/{name}/all_files": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get all files of a dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.File"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets/{namespace}/{name}/dataviewer/catalog": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get catalog of the dataset",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get catalog of the dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets/{namespace}/{name}/dataviewer/rows": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get catalog of the dataset",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get catalog of the dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "config",
+ "name": "config",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "split",
+ "name": "split",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "search",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "where",
+ "name": "where",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "orderby",
+ "name": "orderby",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets/{namespace}/{name}/relations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get dataset related assets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Relations"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/datasets/{namespace}/{name}/viewer/{file_path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get the demo data of the dataset",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Dataset"
+ ],
+ "summary": "Get the demo data of the dataset",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "file_path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/discussions/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "show a discussion",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Show a discussion and its comments",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the discussion id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/component.ShowDiscussionResponse"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a discussion",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Update a discussion",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the discussion id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/component.UpdateDiscussionRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a discussion",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Delete a discussion",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the discussion id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner of the discussion",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/discussions/{id}/comments": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "list discussion comments",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "List discussion comments",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the discussion id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/component.DiscussionResponse_Comment"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new discussion comment",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Create a new discussion comment",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the discussion id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/component.CreateCommentRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/component.CreateCommentResponse"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/evaluations": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Evaluation"
+ ],
+ "summary": "run model evaluation",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body setting of evaluation",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.EvaluationReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/evaluations/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Evaluation"
+ ],
+ "summary": "get model evaluation",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.EvaluationRes"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Evaluation"
+ ],
+ "summary": "delete model evaluation",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/events": {
+ "post": {
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Events"
+ ],
+ "summary": "Report client events",
+ "parameters": [
+ {
+ "description": "Events",
+ "name": "events",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Event"
+ }
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "object"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/jwt/token": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "JWT"
+ ],
+ "summary": "generate jwt token for user",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateJWTReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.CreateJWTResp"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/jwt/{token}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "JWT"
+ ],
+ "summary": "verify jwt token and return user info",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "token",
+ "name": "token",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.User"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/licenses": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List license",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "List license",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "product",
+ "name": "product",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "edition",
+ "name": "edition",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "search",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.License"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Create a license",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Create a license",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateLicenseReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/licenses/license": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get active license status",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Get active license status",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Import a license",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Import a license",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ImportLicenseReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Verify a license",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Verify a license",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ImportLicenseReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/licenses/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get a license by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Get a license by id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "boolean",
+ "default": false,
+ "description": "create license file",
+ "name": "create_license_file",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.License"
+ },
+ "license": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Update a license by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Update a license by id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateLicenseReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Delete a license by id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "License"
+ ],
+ "summary": "Delete a license by id",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/list/datasets_by_path": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "list datasets by paths",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "List"
+ ],
+ "summary": "List datasets by paths",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ListByPathReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.DatasetResp"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/list/models_by_path": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "list models by paths",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "List"
+ ],
+ "summary": "List models by paths",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ListByPathReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.ModelResp"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/list/spaces_by_path": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "List"
+ ],
+ "summary": "List spaces by paths",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ListByPathReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Space"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/mirror/repo": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Create mirror repo",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateMirrorRepoReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/mirror/repos": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Get mirror repos",
+ "parameters": [
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.MirrorRepo"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/mirror/sources": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Get mirror sources",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.MirrorSource"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Create mirror source",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateMirrorSourceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.MirrorSource"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/mirror/sources/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Get mirror source",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.MirrorSource"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Update mirror source",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateMirrorSourceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.MirrorSource"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Delete mirror source",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/mirror/statistics": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Get mirror status counts",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.MirrorStatusCount"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/mirrors": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Mirror"
+ ],
+ "summary": "Get mirrors",
+ "parameters": [
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Mirror"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get visiable models for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Get Visiable models for current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by task tag, deprecated",
+ "name": "task_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by framework tag, deprecated",
+ "name": "framework_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by license tag, deprecated",
+ "name": "license_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by language tag, deprecated",
+ "name": "language_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by tag category",
+ "name": "tag_category",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by tag name",
+ "name": "tag_name",
+ "in": "query"
+ },
+ {
+ "type": "boolean",
+ "default": false,
+ "description": "need op weight",
+ "name": "need_op_weight",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "opencsg",
+ "huggingface",
+ "local"
+ ],
+ "type": "string",
+ "description": "source",
+ "name": "source",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Model"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new model",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Create a new model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateModelReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Model"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/runtime_framework": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List repo runtime framework",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "List repo runtime framework",
+ "parameters": [
+ {
+ "enum": [
+ "models"
+ ],
+ "type": "string",
+ "description": "models",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy_type",
+ "name": "deploy_type",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get model detail",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Get model detail",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "boolean",
+ "default": false,
+ "description": "need op weight",
+ "name": "need_op_weight",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Model"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exists model",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Update a exists model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the model owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateModelReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Model"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exists model",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Delete a exists model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the model owner",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/all_files": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Get all files of a model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.File"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/finetune": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "create a finetune instance",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "deploy setting of instance",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.InstanceRunReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/finetune/{id}": {
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a finetune instance",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Delete a finetune instance",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/finetune/{id}/start": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Start a finetune instance",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Start a finetune instance",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/finetune/{id}/stop": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Stop a finetune instance",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Stop a finetune instance",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/predict": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "invoke model prediction",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Invoke model prediction",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "input for model prediction",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ModelPredictReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/relations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Get model related assets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Relations"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Set dataset relation for model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "set dataset relation",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RelationDatasets"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Relations"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/relations/dataset": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "add dataset relation for model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "add dataset relation",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RelationDataset"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "delete dataset relation for model",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "delelet dataset relation",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RelationDataset"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/run": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "run model as inference",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "deploy setting of inference",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ModelRunReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/run/{id}": {
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a model inference",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Delete a model inference",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/run/{id}/start": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Start a model inference",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Start a model inference",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/run/{id}/stop": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Stop a model inference",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Stop a model inference",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/serverless": {
+ "get": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "get model serverless",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "run model as serverless service",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "deploy setting of serverless",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ModelRunReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/serverless/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get repo serverless detail",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Get repo serverless detail",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Update serverless parameters",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "deploy setting of Serverless",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.DeployUpdateReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/serverless/{id}/logs/{instance}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "get serverless logs",
+ "parameters": [
+ {
+ "enum": [
+ "models"
+ ],
+ "type": "string",
+ "description": "models",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "instance",
+ "name": "instance",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/serverless/{id}/start": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Start a model serverless",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Start a model serverless",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/serverless/{id}/status": {
+ "get": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "get serverless status",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/models/{namespace}/{name}/serverless/{id}/stop": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Stop a model serverless",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Model"
+ ],
+ "summary": "Stop a model serverless",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/namespace/{path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get namespace info",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Namespace",
+ "InternalOnly"
+ ],
+ "summary": "Get namespace info [Internal Only].",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "path",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Namespace"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization info",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Organization"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update organization",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Update organization",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.EditOrgReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Organization"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete organization",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Delete organization",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/codes": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization codes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "page size",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "description": "current page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Code"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/collections": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization Collections",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "page size",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "description": "current page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/datasets": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get organization datasets",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization datasets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "page size",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "description": "current page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/members": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Member"
+ ],
+ "summary": "Get organization members. Org member can get more details.",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ " total": {
+ "type": "integer"
+ },
+ "data": {
+ "$ref": "#/definitions/types.Member"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "user will be added to org with a role",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Member"
+ ],
+ "summary": "Create new membership between org and user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/handler.Create.addMemberRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/members/{username}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Member"
+ ],
+ "summary": "Get user's role in an org",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "user name",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Member"
+ ],
+ "summary": "update user membership",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "user name",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/handler.Update.updateMemberRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "user's role will be remove from org",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Member"
+ ],
+ "summary": "Remove membership between org and user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "user name",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/handler.Delete.removeMemberRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/models": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get organization models",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization models",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "page size",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "description": "current page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Model"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/prompts": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get organization prompts",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization prompts",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "page size",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "description": "current page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.PromptRes"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organization/{namespace}/spaces": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organization Spaces",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "org name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "page size",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "description": "current page number",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Space"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/organizations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get organizations",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Get organizations",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Organization"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new organization",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Organization"
+ ],
+ "summary": "Create a new organization",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "the op user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateOrgReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Organization"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get visiable Prompt repos for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Get Visiable Prompt repos for current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by task tag",
+ "name": "task_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by framework tag",
+ "name": "framework_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by license tag",
+ "name": "license_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by language tag",
+ "name": "language_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "opencsg",
+ "huggingface",
+ "local"
+ ],
+ "type": "string",
+ "description": "source",
+ "name": "source",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.PromptRes"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new prompt repo",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Create a new prompt repo",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreatePromptRepoReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/conversations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List conversations of user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "List conversations of user",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Create new conversation",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Create new conversation",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.Conversation"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/conversations/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get a conversation by uuid",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Get a conversation by uuid",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "conversation uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Update a conversation title",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Update a conversation title",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "conversation uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.ConversationTitle"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Submit a conversation message",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Submit a conversation message",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "conversation uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.Conversation"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Delete a conversation",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Delete a conversation",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "conversation uuid",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/conversations/{id}/message/{msgid}/hate": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Hate a conversation message",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Hate a conversation message",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "conversation uuid",
+ "name": "uuid",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "message id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/conversations/{id}/message/{msgid}/like": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Like a conversation message",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Like a conversation message",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "conversation uuid",
+ "name": "uuid",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "message id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List prompts",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "List prompts",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exists prompt repo",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Update a exists prompt repo",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdatePromptRepoReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exists prompt repo",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Delete a exists prompt repo",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/branches": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Get the branches of prompt repository",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Branch"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/prompt/file": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Create prompt in repo",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Create prompt in repo",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/prompt/file/{file_path}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Update prompt in repo",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Update prompt in repo",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the file relative path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Delete prompt in repo",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Delete prompt in repo",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the file relative path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/prompt/view/{file_path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get prompts by file",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Get prompts by file",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the file relative path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/relations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Get prompt related assets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Relations"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Set model relation for prompt",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "set model relation",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RelationModels"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/relations/model": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "add model relation for prompt",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "add model relation",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RelationModel"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "delete model relation for prompt",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "delelet model relation",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RelationModel"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/tags": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "Get the tags of prompt repository",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.Tag"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/prompts/{namespace}/{name}/tags/{category}": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Prompt"
+ ],
+ "summary": "update the tags of a certain category",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "enum": [
+ "task",
+ "license",
+ "framework",
+ "language",
+ "industry"
+ ],
+ "type": "string",
+ "description": "tag category",
+ "name": "category",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "tag names in array",
+ "name": "tags",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/recom/opweight": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Recommendation"
+ ],
+ "summary": "set op weight for repo recommendation",
+ "parameters": [
+ {
+ "description": "json request body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/handler.SetOpWeight.SetOpWeightReq"
+ }
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/runtime_framework": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get all runtime frameworks for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Get all runtime frameworks for current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/runtime_framework/models": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get visible models for all runtime frameworks for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Get Visible models for all runtime frameworks for current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ },
+ {
+ "enum": [
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy_type",
+ "name": "deploy_type",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/runtime_framework/{id}": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "set model runtime frameworks",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Set model runtime frameworks",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy_type",
+ "name": "deploy_type",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFrameworkModels"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "set model runtime frameworks",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Set model runtime frameworks",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy_type",
+ "name": "deploy_type",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFrameworkModels"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/runtime_framework/{id}/architecture": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get runtime framework architectures",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Get runtime framework architectures",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "set runtime framework architectures",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Set runtime framework architectures",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeArchitecture"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Delete runtime framework architectures",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Delete runtime framework architectures",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeArchitecture"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/runtime_framework/{id}/models": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get visible models by runtime framework for current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Get Visible models by runtime framework for current user",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy_type",
+ "name": "deploy_type",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/runtime_framework/{id}/scan": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Scan runtime architecture",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "RuntimeFramework"
+ ],
+ "summary": "Scan runtime architecture",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "runtime framework id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "description": "scan_type(0:all models, 1:new models, 2:old models)",
+ "name": "scan_type",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFrameworkModels"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/space_resources": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get space resources",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceReource"
+ ],
+ "summary": "Get space resources",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "cluster_id",
+ "name": "cluster_id",
+ "in": "query"
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy type(0-space,1-inference,2-finetune)",
+ "name": "deploy_type",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.SpaceResource"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create space resource",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceReource"
+ ],
+ "summary": "Create space resource",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateSpaceResourceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.SpaceResource"
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/space_resources/{id}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exist space resource",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceReource"
+ ],
+ "summary": "Update a exist space resource",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateSpaceResourceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.SpaceResource"
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exist space resource",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceReource"
+ ],
+ "summary": "Delete a exist space resource",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/space_sdks": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get space sdks",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceSdk"
+ ],
+ "summary": "Get space sdks",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.SpaceSdk"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create space sdk",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceSdk"
+ ],
+ "summary": "Create space sdk",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateSpaceSdkReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.SpaceSdk"
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/space_sdks/{id}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exist space sdk",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceSdk"
+ ],
+ "summary": "Update a exist space sdk",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateSpaceSdkReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.SpaceSdk"
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exist space sdk",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SpaceSdk"
+ ],
+ "summary": "Delete a exist space sdk",
+ "parameters": [
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "写id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get spaces visible to current user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "Get spaces visible to current user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "search text",
+ "name": "search",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by task tag",
+ "name": "task_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by framework tag",
+ "name": "framework_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by license tag",
+ "name": "license_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "filter by language tag",
+ "name": "language_tag",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "sort by",
+ "name": "sort",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "opencsg",
+ "huggingface",
+ "local"
+ ],
+ "type": "string",
+ "description": "source",
+ "name": "source",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Space"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new space",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "Create a new space",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateSpaceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Space"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces/{namespace}/{name}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "show space detail",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Space"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update a exists space",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "Update a exists space",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateSpaceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Space"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exists space",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "Delete a exists space",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces/{namespace}/{name}/logs": {
+ "get": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "get space logs",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces/{namespace}/{name}/run": {
+ "post": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "run space app",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces/{namespace}/{name}/status": {
+ "get": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "get space status",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces/{namespace}/{name}/stop": {
+ "post": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "stop space app",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/spaces/{namespace}/{name}/wakeup": {
+ "post": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Space"
+ ],
+ "summary": "wake up space app",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/sync/client_setting": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get sync client setting",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Sync"
+ ],
+ "summary": "Get sync client setting",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.SyncClientSetting"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Create sync client setting or update an existing sync client setting",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Sync"
+ ],
+ "summary": "Create sync client setting or update an existing sync client setting",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateSyncClientSettingReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.SyncClientSetting"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/sync/version/latest": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Sync"
+ ],
+ "summary": "Get latest version",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current version",
+ "name": "cur",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.SyncVersionResponse"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/tags": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get all tags",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Tag"
+ ],
+ "summary": "Get all tags",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "category name",
+ "name": "category",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "model",
+ "dataset"
+ ],
+ "type": "string",
+ "description": "scope name",
+ "name": "scope",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "tags",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.Tag"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/telemetry/usage": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Telemetry"
+ ],
+ "summary": "Submit telemetry data for a client",
+ "parameters": [
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/telemetry.Usage"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/token/{app}/{token_name}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "Refresh a access token for a user",
+ "parameters": [
+ {
+ "enum": [
+ "git",
+ "starship"
+ ],
+ "type": "string",
+ "description": "application",
+ "name": "app",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "token_name",
+ "name": "token_name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "new expire time, in format RFC3339, like 2006-01-02T15:04:05Z07:00",
+ "name": "expired_at",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "Create access token for an special application",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "token name",
+ "name": "token_name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ "git",
+ "starship"
+ ],
+ "type": "string",
+ "description": "application",
+ "name": "app",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateUserTokenRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.AccessToken"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "Delete access token of a app",
+ "parameters": [
+ {
+ "enum": [
+ "git",
+ "starship"
+ ],
+ "type": "string",
+ "description": "application",
+ "name": "app",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "token_name",
+ "name": "token_name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/token/{token_value}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "Get token and owner's detail by the token value",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "token_value",
+ "name": "token_value",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ "git",
+ "starship"
+ ],
+ "type": "string",
+ "description": "application",
+ "name": "app",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user info. Admin and the user self can see full info, other users can only see basic info.",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username or uuid, defined by the query string 'type'",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "username",
+ "uuid"
+ ],
+ "type": "string",
+ "description": "path param is usernam or uuid, default to username",
+ "name": "type",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.User"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "update user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Update user. If change user name, should only send 'new_username' in the request body.",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateUserRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Delete user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateUserRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/codes": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user codes",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user codes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Code"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/collections": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user's collections",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/datasets": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user datasets",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user datasets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/evaluations": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user evaluations",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user evaluations",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/finetune/instances": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get user running notebook instances",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user running notebook instances",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "page index",
+ "name": "page",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/codes": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user likes codes",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user likes codes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Code"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/collections": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user likes collections",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Collection"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/collections/{id}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Add collection likes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "collection id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "delete collection likes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "collection id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/datasets": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user likes datasets",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user likes datasets",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/models": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user likes models",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user likes models",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Model"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/spaces": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user likes spaces",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Space"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/likes/{repoid}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Add user likes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo id",
+ "name": "repo_id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Delete user likes",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Delete user likes",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo id",
+ "name": "repo_id",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/models": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user models",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user models",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Model"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/order/resource": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create order for user's resource",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "create order for user's resource",
+ "parameters": [
+ {
+ "description": "create order request",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateUserResourceReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/order/resources": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user's resource",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "get user's resource",
+ "parameters": [
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.UserResourcesResp"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/order/resources/{id}": {
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete user's resource by order detail id",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "delete user's resource by order detail id",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/prompts": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get user prompts",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user prompts",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/run/serverless": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get serverless deploys",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get serverless deploys",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "page index",
+ "name": "page",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/run/{repo_type}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get user running deploys",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user running deploys",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ "model",
+ "space"
+ ],
+ "type": "string",
+ "description": "model,space",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy type(0-space,1-inference,2-finetune)",
+ "name": "deploy_type",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 50,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "page index",
+ "name": "page",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/spaces": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get user spaces",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Space"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/ssh_key/{name}": {
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete specific SSH key for the given user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SSH Key"
+ ],
+ "summary": "Delete specific SSH key for the given user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "key name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/ssh_keys": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "get all SSH keys for the given user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SSH Key"
+ ],
+ "summary": "Get all SSH keys for the given user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.SSHKey"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new SSH key for the given user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "SSH Key"
+ ],
+ "summary": "Create a new SSH key for the given user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateSSHKeyRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.SSHKey"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/tokens": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "Get all access tokens for a user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "git",
+ "starship"
+ ],
+ "type": "string",
+ "description": "application",
+ "name": "app",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create access token for a user",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "[Deprecated: use POST:/token/{app}/{username} instead]",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateUserTokenRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.AccessToken"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/tokens/first": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "Get or create first available access token for a user",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "git",
+ "starship"
+ ],
+ "type": "string",
+ "description": "application",
+ "name": "app",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "token name",
+ "name": "token_name",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/user/{username}/tokens/{token_name}": {
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Access token"
+ ],
+ "summary": "[Deprecated: use DELETE:/token/{app}/{token_name} instead]",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "username",
+ "name": "username",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "token_name",
+ "name": "token_name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.DeleteUserTokenRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/users": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "User"
+ ],
+ "summary": "Get users info. Only Admin",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "search",
+ "name": "search",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.User"
+ }
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/blob/{file_path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey or JWT": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get the repo file information like size, content, sha etc",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "file path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.File"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/branches": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get the branches of repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Branch"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/commit/{commit_id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get commit diff of repository and data field of response need to be decode with base64",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "commit id",
+ "name": "commit_id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.CommitResponse"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/commits": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get all commits of repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 20,
+ "description": "per",
+ "name": "per",
+ "in": "query"
+ },
+ {
+ "type": "integer",
+ "default": 1,
+ "description": "per page",
+ "name": "page",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Commit"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/discussions": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "list repo discussions",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "List repo discussions",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "repository type",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/component.ListRepoDiscussionResponse"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create a new repo discussion",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Discussion"
+ ],
+ "summary": "Create a new repo discussion",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "current user, the owner",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "repository type",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/component.CreateRepoDiscussionRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/component.CreateDiscussionResponse"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/download/{file_path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey or JWT": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Download a repo file [Depricated: use 'resolve' api instead]",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "file path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "file name to save as",
+ "name": "save_as",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "object"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/incr_downloads": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Increase repo download count by 1",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/last_commit": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get the last commit of repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.Commit"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/mirror": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get a mirror",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Mirror"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Update a mirror for a existing repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateMirrorParams"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Mirror"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Create mirror for a existing repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateMirrorParams"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Mirror"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Delete a mirror",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/mirror/progress": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get Mirror sync progress",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.LFSSyncProgressResp"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/mirror/sync": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Triggers the mirror synchronization",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/mirror_from_saas": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Mirror repo from OpenCSG Saas(only on-premises)",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,datasets,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.Response"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/database.Mirror"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/raw/{file_path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get the last commit of repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "file path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Update existing file in repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the new file relative path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "create file request",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.UpdateFileReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.UpdateFileResp"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Create a new file in repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "the new file relative path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "create file request",
+ "name": "req",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.CreateFileReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "$ref": "#/definitions/types.CreateFileResp"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/resolve/{file_path}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Download a rep file",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "file path",
+ "name": "file_path",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "string"
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/run": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List repo deploys",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "List repo deploys",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/run/{id}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Get repo deploy detail",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get repo deploy detail",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "401": {
+ "description": "Permission denied",
+ "schema": {
+ "$ref": "#/definitions/types.APIUnauthorized"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Update deploy parameters",
+ "parameters": [
+ {
+ "enum": [
+ "models"
+ ],
+ "type": "string",
+ "description": "models",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "description": "deploy setting of inference",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.DeployUpdateReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/run/{id}/logs/{instance}": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "get deploy instance logs",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "instance",
+ "name": "instance",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "401": {
+ "description": "Permission denied",
+ "schema": {
+ "$ref": "#/definitions/types.APIUnauthorized"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/run/{id}/status": {
+ "get": {
+ "security": [
+ {
+ "JWT token": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "get deploy status",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "deploy id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current_user",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ }
+ ],
+ "responses": {
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "401": {
+ "description": "Permission denied",
+ "schema": {
+ "$ref": "#/definitions/types.APIUnauthorized"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/runtime_framework": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "List repo runtime framework",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "List repo runtime framework",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "enum": [
+ 0,
+ 1,
+ 2,
+ 4
+ ],
+ "type": "integer",
+ "default": 1,
+ "description": "deploy_type",
+ "name": "deploy_type",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "type": "string"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "create runtime framework",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Create runtime framework",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFrameworkReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFramework"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/runtime_framework/{id}": {
+ "put": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "Update runtime framework",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Update runtime framework",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ },
+ {
+ "description": "body",
+ "name": "body",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFrameworkReq"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.RuntimeFramework"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ },
+ "delete": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "description": "delete a exist RuntimeFramework",
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Delete a exist RuntimeFramework",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "namespace",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "integer",
+ "description": "id",
+ "name": "id",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/tags": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get the tags of repository",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.Tag"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/tags/{category}": {
+ "post": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "update the tags of a certain category",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "current user name",
+ "name": "current_user",
+ "in": "query",
+ "required": true
+ },
+ {
+ "enum": [
+ "task",
+ "license",
+ "framework",
+ "language",
+ "industry"
+ ],
+ "type": "string",
+ "description": "tag category",
+ "name": "category",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "tag names in array",
+ "name": "tags",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/types.Response"
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ },
+ "/{repo_type}/{namespace}/{name}/tree": {
+ "get": {
+ "security": [
+ {
+ "ApiKey": []
+ }
+ ],
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "Repository"
+ ],
+ "summary": "Get repository file tree",
+ "parameters": [
+ {
+ "enum": [
+ "models",
+ "datasets",
+ "codes",
+ "spaces"
+ ],
+ "type": "string",
+ "description": "models,dataset,codes or spaces",
+ "name": "repo_type",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo owner name",
+ "name": "namespace",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "repo name",
+ "name": "name",
+ "in": "path",
+ "required": true
+ },
+ {
+ "type": "string",
+ "description": "root dir",
+ "name": "path",
+ "in": "query"
+ },
+ {
+ "type": "string",
+ "description": "branch or tag",
+ "name": "ref",
+ "in": "query"
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ResponseWithTotal"
+ },
+ {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.File"
+ }
+ }
+ }
+ }
+ ]
+ }
+ },
+ "400": {
+ "description": "Bad request",
+ "schema": {
+ "$ref": "#/definitions/types.APIBadRequest"
+ }
+ },
+ "500": {
+ "description": "Internal server error",
+ "schema": {
+ "$ref": "#/definitions/types.APIInternalServerError"
+ }
+ }
+ }
+ }
+ }
+ },
+ "definitions": {
+ "component.CreateCommentRequest": {
+ "type": "object",
+ "required": [
+ "content"
+ ],
+ "properties": {
+ "commentable_id": {
+ "type": "integer"
+ },
+ "commentable_type": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ }
+ }
+ },
+ "component.CreateCommentResponse": {
+ "type": "object",
+ "properties": {
+ "commentable_id": {
+ "type": "integer"
+ },
+ "commentable_type": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "user": {
+ "$ref": "#/definitions/component.DiscussionResponse_User"
+ }
+ }
+ },
+ "component.CreateDiscussionResponse": {
+ "type": "object",
+ "properties": {
+ "comment_count": {
+ "description": "DiscussionableID int64 ` + "`" + `json:\"discussionable_id\"` + "`" + `\nDiscussionableType string ` + "`" + `json:\"discussionable_type\"` + "`" + `",
+ "type": "integer"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "title": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/component.DiscussionResponse_User"
+ }
+ }
+ },
+ "component.CreateRepoDiscussionRequest": {
+ "type": "object",
+ "required": [
+ "title"
+ ],
+ "properties": {
+ "title": {
+ "type": "string"
+ }
+ }
+ },
+ "component.DiscussionResponse_Comment": {
+ "type": "object",
+ "properties": {
+ "content": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "user": {
+ "$ref": "#/definitions/component.DiscussionResponse_User"
+ }
+ }
+ },
+ "component.DiscussionResponse_User": {
+ "type": "object",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "component.ListRepoDiscussionResponse": {
+ "type": "object",
+ "properties": {
+ "discussions": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/component.CreateDiscussionResponse"
+ }
+ }
+ }
+ },
+ "component.ShowDiscussionResponse": {
+ "type": "object",
+ "properties": {
+ "comment_count": {
+ "type": "integer"
+ },
+ "comments": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/component.DiscussionResponse_Comment"
+ }
+ },
+ "id": {
+ "type": "integer"
+ },
+ "title": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/component.DiscussionResponse_User"
+ }
+ }
+ },
+ "component.UpdateCommentRequest": {
+ "type": "object",
+ "required": [
+ "content"
+ ],
+ "properties": {
+ "content": {
+ "type": "string"
+ }
+ }
+ },
+ "component.UpdateDiscussionRequest": {
+ "type": "object",
+ "required": [
+ "title"
+ ],
+ "properties": {
+ "title": {
+ "type": "string"
+ }
+ }
+ },
+ "consts.PaymentChannel": {
+ "type": "string",
+ "enum": [
+ "alipay",
+ "alipay_wap",
+ "alipay_qr",
+ "alipay_scan",
+ "alipay_lite",
+ "alipay_pc_direct",
+ "wx",
+ "wx_pub",
+ "wx_pub_qr",
+ "wx_pub_scan",
+ "wx_wap",
+ "wx_lite",
+ "qpay",
+ "qpay_pub",
+ "upacp",
+ "upacp_pc",
+ "upacp_qr",
+ "upacp_scan",
+ "upacp_wap",
+ "upacp_b2b",
+ "cp_b2b",
+ "applepay_upacp",
+ "cmb_wallet",
+ "cmb_pc_qr",
+ "bfb_wap",
+ "jdpay_wap",
+ "yeepay_wap",
+ "isv_qr",
+ "isv_scan",
+ "isv_wap",
+ "isv_lite",
+ "ccb_pay",
+ "ccb_qr",
+ "cmpay",
+ "coolcredit",
+ "cb_alipay",
+ "cb_alipay_wap",
+ "cb_alipay_qr",
+ "cb_alipay_scan",
+ "cb_alipay_pc_direct",
+ "cb_wx",
+ "cb_wx_pub",
+ "cb_wx_pub_qr",
+ "cb_wx_pub_scan",
+ "paypal",
+ "balance",
+ "yeepay_wx_pub_qr",
+ "yeepay_wx_pub",
+ "yeepay_wx_pub_ofl",
+ "yeepay_wx_lite",
+ "yeepay_wx_lite_ofl",
+ "yeepay_wx_pub_scan",
+ "yeepay_alipay_qr",
+ "yeepay_alipay_lite",
+ "yeepay_alipay_pub",
+ "yeepay_alipay_scan",
+ "yeepay_upacp_qr",
+ "yeepay_upacp_scan"
+ ],
+ "x-enum-varnames": [
+ "ChannelAlipay",
+ "ChannelAlipayWap",
+ "ChannelAlipayQr",
+ "ChannelAlipayScan",
+ "ChannelAlipayLite",
+ "ChannelAlipayPcDirect",
+ "ChannelWx",
+ "ChannelWxPub",
+ "ChannelWxPubQr",
+ "ChannelWxPubScan",
+ "ChannelWxWap",
+ "ChannelWxLite",
+ "ChannelQpay",
+ "ChannelQpayPub",
+ "ChannelUpacp",
+ "ChannelUpacpPc",
+ "ChannelUpacpQr",
+ "ChannelUpacpScan",
+ "ChannelUpacpWap",
+ "ChannelUpacpB2b",
+ "ChannelCpB2b",
+ "ChannelApplepayUpacp",
+ "ChannelCmbWallet",
+ "ChannelCmbPcQr",
+ "ChannelBfbWap",
+ "ChannelJdpayWap",
+ "ChannelYeepayWap",
+ "ChannelIsvQr",
+ "ChannelIsvScan",
+ "ChannelIsvWap",
+ "ChannelIsvLite",
+ "ChannelCcbPay",
+ "ChannelCcbQr",
+ "ChannelCmpay",
+ "ChannelCoolcredit",
+ "ChannelCbAlipay",
+ "ChannelCbAlipayWap",
+ "ChannelCbAlipayQr",
+ "ChannelCbAlipayScan",
+ "ChannelCbAlipayPcDirect",
+ "ChannelCbWx",
+ "ChannelCbWxPub",
+ "ChannelCbWxPubQr",
+ "ChannelCbWxPubScan",
+ "ChannelPaypal",
+ "ChannelBalance",
+ "ChannelYeepayWxPubQr",
+ "ChannelYeepayWxPub",
+ "ChannelYeepayWxPubOfl",
+ "ChannelYeepayWxLite",
+ "ChannelYeepayWxLiteOfl",
+ "ChannelYeepayWxPubScan",
+ "ChannelYeepayAlipayQr",
+ "ChannelYeepayAlipayLite",
+ "ChannelYeepayAlipayPub",
+ "ChannelYeepayAlipayScan",
+ "ChannelYeepayUpacpQr",
+ "ChannelYeepayUpacpScan"
+ ]
+ },
+ "database.AccessToken": {
+ "type": "object",
+ "properties": {
+ "application": {
+ "description": "example: csghub, starship",
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.AccessTokenApp"
+ }
+ ]
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "expired_at": {
+ "type": "string"
+ },
+ "git_id": {
+ "type": "integer"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "is_active": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "string"
+ },
+ "permission": {
+ "type": "string"
+ },
+ "token": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/database.User"
+ },
+ "user_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "database.Code": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "last_updated_at": {
+ "type": "string"
+ },
+ "repository": {
+ "$ref": "#/definitions/database.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.Collection": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "repositories": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.Repository"
+ }
+ },
+ "theme": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user_id": {
+ "type": "integer"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "database.Dataset": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "last_updated_at": {
+ "type": "string"
+ },
+ "repository": {
+ "$ref": "#/definitions/database.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.License": {
+ "type": "object",
+ "properties": {
+ "company": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "edition": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "expire_time": {
+ "type": "string"
+ },
+ "extra": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "key": {
+ "type": "string"
+ },
+ "max_user": {
+ "type": "integer"
+ },
+ "product": {
+ "type": "string"
+ },
+ "remark": {
+ "type": "string"
+ },
+ "start_time": {
+ "type": "string"
+ },
+ "status": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user_uuid": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "database.Mirror": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "interval": {
+ "type": "string"
+ },
+ "last_message": {
+ "type": "string"
+ },
+ "last_updated_at": {
+ "type": "string"
+ },
+ "local_repo_path": {
+ "type": "string"
+ },
+ "mirror_source": {
+ "$ref": "#/definitions/database.MirrorSource"
+ },
+ "mirror_source_id": {
+ "type": "integer"
+ },
+ "mirror_task_id": {
+ "type": "integer"
+ },
+ "next_execution_timestamp": {
+ "type": "string"
+ },
+ "priority": {
+ "$ref": "#/definitions/types.MirrorPriority"
+ },
+ "progress": {
+ "type": "integer"
+ },
+ "push_mirror_created": {
+ "type": "boolean"
+ },
+ "repository": {
+ "$ref": "#/definitions/database.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "source_repo_path": {
+ "type": "string"
+ },
+ "source_url": {
+ "type": "string"
+ },
+ "status": {
+ "$ref": "#/definitions/types.MirrorTaskStatus"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.MirrorSource": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "info_api_url": {
+ "type": "string"
+ },
+ "source_name": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.Model": {
+ "type": "object",
+ "properties": {
+ "base_model": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "last_updated_at": {
+ "type": "string"
+ },
+ "repository": {
+ "$ref": "#/definitions/database.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.Namespace": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "mirrored": {
+ "type": "boolean"
+ },
+ "namespace_type": {
+ "$ref": "#/definitions/database.NamespaceType"
+ },
+ "path": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/database.User"
+ },
+ "user_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "database.NamespaceType": {
+ "type": "string",
+ "enum": [
+ "user",
+ "organization"
+ ],
+ "x-enum-varnames": [
+ "UserNamespace",
+ "OrgNamespace"
+ ]
+ },
+ "database.Organization": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "git_path": {
+ "type": "string"
+ },
+ "homepage": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "logo": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "$ref": "#/definitions/database.Namespace"
+ },
+ "namespace_id": {
+ "type": "integer"
+ },
+ "org_type": {
+ "type": "string"
+ },
+ "path": {
+ "description": "unique name of the organization",
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/database.User"
+ },
+ "user_id": {
+ "type": "integer"
+ },
+ "verified": {
+ "type": "boolean"
+ }
+ }
+ },
+ "database.Repository": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "download_count": {
+ "type": "integer"
+ },
+ "downloads": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.RepositoryDownload"
+ }
+ },
+ "git_path": {
+ "type": "string"
+ },
+ "http_clone_url": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "labels": {
+ "description": "Depreated",
+ "type": "string"
+ },
+ "license": {
+ "type": "string"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "mirror": {
+ "$ref": "#/definitions/database.Mirror"
+ },
+ "name": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "description": "Depreated",
+ "type": "string"
+ },
+ "repository_type": {
+ "$ref": "#/definitions/types.RepositoryType"
+ },
+ "sensitive_check_status": {
+ "$ref": "#/definitions/types.SensitiveCheckStatus"
+ },
+ "source": {
+ "$ref": "#/definitions/types.RepositorySource"
+ },
+ "ssh_clone_url": {
+ "type": "string"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.Tag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/database.User"
+ },
+ "user_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "database.RepositoryDownload": {
+ "type": "object",
+ "properties": {
+ "click_download_count": {
+ "type": "integer"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "date": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "repository": {
+ "$ref": "#/definitions/database.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "database.SSHKey": {
+ "type": "object",
+ "properties": {
+ "content": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "fingerprint_sha256": {
+ "type": "string"
+ },
+ "git_id": {
+ "type": "integer"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/database.User"
+ },
+ "user_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "database.SyncClientSetting": {
+ "type": "object",
+ "properties": {
+ "concurrent_count": {
+ "type": "integer"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "default": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "max_bandwidth": {
+ "type": "integer"
+ },
+ "token": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.Tag": {
+ "type": "object",
+ "properties": {
+ "built_in": {
+ "type": "boolean"
+ },
+ "category": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "group": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "scope": {
+ "$ref": "#/definitions/database.TagScope"
+ },
+ "show_name": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "database.TagScope": {
+ "type": "string",
+ "enum": [
+ "model",
+ "dataset",
+ "code",
+ "space",
+ "prompt"
+ ],
+ "x-enum-varnames": [
+ "ModelTagScope",
+ "DatasetTagScope",
+ "CodeTagScope",
+ "SpaceTagScope",
+ "PromptTagScope"
+ ]
+ },
+ "database.User": {
+ "type": "object",
+ "properties": {
+ "accessTokens": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.AccessToken"
+ }
+ },
+ "avatar": {
+ "type": "string"
+ },
+ "bio": {
+ "type": "string"
+ },
+ "can_change_username": {
+ "description": "allow user to change username once",
+ "type": "boolean"
+ },
+ "company_verified": {
+ "type": "boolean"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "email_verified": {
+ "type": "boolean"
+ },
+ "gender": {
+ "type": "string"
+ },
+ "git_id": {
+ "type": "integer"
+ },
+ "homepage": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "last_login_at": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/database.Namespace"
+ }
+ },
+ "password_hash": {
+ "description": "password for user registered without casdoor",
+ "type": "string"
+ },
+ "phone": {
+ "type": "string"
+ },
+ "phone_verified": {
+ "type": "boolean"
+ },
+ "reg_provider": {
+ "description": "user registered from default login page, from casdoor, etc. Possible values:\n\n- \"default\"\n- \"casdoor\"",
+ "type": "string"
+ },
+ "role_mask": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ },
+ "uuid": {
+ "description": "TODO:add unique index after migration",
+ "type": "string"
+ }
+ }
+ },
+ "handler.Create.addMemberRequest": {
+ "type": "object",
+ "required": [
+ "role",
+ "users"
+ ],
+ "properties": {
+ "role": {
+ "type": "string"
+ },
+ "users": {
+ "description": "name of user will be added to the org as a member",
+ "type": "string",
+ "example": "user1,user2"
+ }
+ }
+ },
+ "handler.Delete.removeMemberRequest": {
+ "type": "object",
+ "required": [
+ "role"
+ ],
+ "properties": {
+ "role": {
+ "type": "string"
+ }
+ }
+ },
+ "handler.SetOpWeight.SetOpWeightReq": {
+ "type": "object",
+ "required": [
+ "repo_id",
+ "weight"
+ ],
+ "properties": {
+ "repo_id": {
+ "type": "integer"
+ },
+ "weight": {
+ "type": "integer"
+ }
+ }
+ },
+ "handler.Update.updateMemberRequest": {
+ "type": "object",
+ "required": [
+ "new_role",
+ "old_role"
+ ],
+ "properties": {
+ "new_role": {
+ "type": "string"
+ },
+ "old_role": {
+ "type": "string"
+ }
+ }
+ },
+ "telemetry.Counts": {
+ "type": "object",
+ "properties": {
+ "codes": {
+ "type": "integer"
+ },
+ "datasets": {
+ "type": "integer"
+ },
+ "models": {
+ "type": "integer"
+ },
+ "spaces": {
+ "type": "integer"
+ },
+ "total_repos": {
+ "type": "integer"
+ }
+ }
+ },
+ "telemetry.Licensee": {
+ "type": "object",
+ "properties": {
+ "Company": {
+ "type": "string"
+ },
+ "Email": {
+ "type": "string"
+ },
+ "Name": {
+ "type": "string"
+ }
+ }
+ },
+ "telemetry.Settings": {
+ "type": "object",
+ "properties": {
+ "collected_data_categories": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "gitaly_apdex": {
+ "type": "number"
+ },
+ "ldap_encrypted_secrets_enabled": {
+ "type": "boolean"
+ },
+ "operating_system": {
+ "type": "string"
+ },
+ "service_ping_features_enabled": {
+ "type": "boolean"
+ },
+ "smtp_encrypted_secrets_enabled": {
+ "type": "boolean"
+ },
+ "snowplow_configured_to_gitlab_collector": {
+ "type": "boolean"
+ },
+ "snowplow_enabled": {
+ "type": "boolean"
+ }
+ }
+ },
+ "telemetry.Usage": {
+ "type": "object",
+ "properties": {
+ "active_user_count": {
+ "type": "integer"
+ },
+ "counts": {
+ "$ref": "#/definitions/telemetry.Counts"
+ },
+ "edition": {
+ "type": "string"
+ },
+ "historical_max_users": {
+ "type": "integer"
+ },
+ "hostname": {
+ "type": "string"
+ },
+ "installation_type": {
+ "type": "string"
+ },
+ "license_add_ons": {
+ "type": "object",
+ "additionalProperties": true
+ },
+ "license_billable_users": {
+ "type": "integer"
+ },
+ "license_expires_at": {
+ "type": "string"
+ },
+ "license_id": {
+ "type": "integer"
+ },
+ "license_md5": {
+ "type": "string"
+ },
+ "license_plan": {
+ "type": "string"
+ },
+ "license_starts_at": {
+ "type": "string"
+ },
+ "license_user_count": {
+ "type": "integer"
+ },
+ "licensee": {
+ "$ref": "#/definitions/telemetry.Licensee"
+ },
+ "recorded_at": {
+ "type": "string"
+ },
+ "settings": {
+ "$ref": "#/definitions/telemetry.Settings"
+ },
+ "uuid": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.ACCT_QUOTA_REQ": {
+ "type": "object",
+ "properties": {
+ "repo_count_limit": {
+ "type": "integer"
+ },
+ "speed_limit": {
+ "type": "integer"
+ },
+ "traffic_limit": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.ACCT_QUOTA_STATEMENT_REQ": {
+ "type": "object",
+ "properties": {
+ "repo_path": {
+ "type": "string"
+ },
+ "repo_type": {
+ "type": "string"
+ }
+ }
+ },
+ "types.APIBadRequest": {
+ "type": "object"
+ },
+ "types.APIInternalServerError": {
+ "type": "object"
+ },
+ "types.APIUnauthorized": {
+ "type": "object"
+ },
+ "types.AccessTokenApp": {
+ "type": "string",
+ "enum": [
+ "git",
+ "git",
+ "mirror",
+ "starship"
+ ],
+ "x-enum-varnames": [
+ "AccessTokenAppGit",
+ "AccessTokenAppCSGHub",
+ "AccessTokenAppMirror",
+ "AccessTokenAppStarship"
+ ]
+ },
+ "types.AcctOrderDetailReq": {
+ "type": "object",
+ "required": [
+ "order_count",
+ "resource_id",
+ "sku_type",
+ "sku_unit_type"
+ ],
+ "properties": {
+ "begin_time": {
+ "type": "string"
+ },
+ "order_count": {
+ "type": "integer",
+ "minimum": 1
+ },
+ "resource_id": {
+ "type": "string"
+ },
+ "sku_type": {
+ "type": "integer"
+ },
+ "sku_unit_type": {
+ "type": "string"
+ }
+ }
+ },
+ "types.AcctPriceCreateReq": {
+ "type": "object",
+ "properties": {
+ "quota": {
+ "type": "string"
+ },
+ "resource_id": {
+ "type": "string"
+ },
+ "sku_desc": {
+ "type": "string"
+ },
+ "sku_kind": {
+ "type": "integer"
+ },
+ "sku_price": {
+ "type": "integer"
+ },
+ "sku_price_currency": {
+ "type": "string"
+ },
+ "sku_price_id": {
+ "type": "integer"
+ },
+ "sku_type": {
+ "type": "integer"
+ },
+ "sku_unit": {
+ "type": "integer"
+ },
+ "sku_unit_type": {
+ "type": "string"
+ }
+ }
+ },
+ "types.AcctRechargeReq": {
+ "type": "object",
+ "properties": {
+ "channelCode": {
+ "$ref": "#/definitions/consts.PaymentChannel"
+ },
+ "rechargeAmount": {
+ "description": "unit yuan",
+ "type": "number"
+ }
+ }
+ },
+ "types.AcctRechargeResp": {
+ "type": "object",
+ "properties": {
+ "channel": {
+ "$ref": "#/definitions/consts.PaymentChannel"
+ },
+ "content": {
+ "type": "string"
+ },
+ "createTime": {
+ "description": "2024-11-18 15:50:47",
+ "type": "string"
+ },
+ "orderNo": {
+ "type": "string"
+ },
+ "rechargeUUID": {
+ "type": "string"
+ }
+ }
+ },
+ "types.Branch": {
+ "type": "object",
+ "properties": {
+ "commit": {
+ "$ref": "#/definitions/types.RepoBranchCommit"
+ },
+ "message": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ }
+ }
+ },
+ "types.Code": {
+ "type": "object",
+ "properties": {
+ "can_manage": {
+ "type": "boolean"
+ },
+ "can_write": {
+ "type": "boolean"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "downloads": {
+ "type": "integer"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "license": {
+ "type": "string"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "$ref": "#/definitions/types.Namespace"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "repository": {
+ "$ref": "#/definitions/types.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "sensitive_check_status": {
+ "type": "string"
+ },
+ "source": {
+ "$ref": "#/definitions/types.RepositorySource"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/types.User"
+ },
+ "user_likes": {
+ "type": "boolean"
+ }
+ }
+ },
+ "types.Collection": {
+ "type": "object",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "can_manage": {
+ "type": "boolean"
+ },
+ "can_write": {
+ "type": "boolean"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "repositories": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.CollectionRepository"
+ }
+ },
+ "theme": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user_likes": {
+ "type": "boolean"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CollectionRepository": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "download_count": {
+ "type": "integer"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "repository_type": {
+ "$ref": "#/definitions/types.RepositoryType"
+ },
+ "status": {
+ "type": "string"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.Commit": {
+ "type": "object",
+ "properties": {
+ "author_email": {
+ "type": "string"
+ },
+ "author_name": {
+ "type": "string"
+ },
+ "authored_date": {
+ "type": "string"
+ },
+ "committer_date": {
+ "type": "string"
+ },
+ "committer_email": {
+ "type": "string"
+ },
+ "committer_name": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "id": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CommitMeta": {
+ "type": "object",
+ "properties": {
+ "sha": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CommitResponse": {
+ "type": "object",
+ "properties": {
+ "author_email": {
+ "type": "string"
+ },
+ "author_name": {
+ "type": "string"
+ },
+ "authored_date": {
+ "type": "string"
+ },
+ "committer_date": {
+ "type": "string"
+ },
+ "committer_email": {
+ "type": "string"
+ },
+ "committer_name": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "diff": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "files": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "id": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "parents": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.CommitMeta"
+ }
+ },
+ "stats": {
+ "$ref": "#/definitions/types.CommitStats"
+ }
+ }
+ },
+ "types.CommitStats": {
+ "type": "object",
+ "properties": {
+ "additions": {
+ "type": "integer"
+ },
+ "deletions": {
+ "type": "integer"
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.Conversation": {
+ "type": "object",
+ "required": [
+ "message",
+ "uuid"
+ ],
+ "properties": {
+ "message": {
+ "type": "string"
+ },
+ "temperature": {
+ "type": "number"
+ },
+ "uuid": {
+ "type": "string"
+ }
+ }
+ },
+ "types.ConversationTitle": {
+ "type": "object",
+ "required": [
+ "title",
+ "uuid"
+ ],
+ "properties": {
+ "title": {
+ "type": "string"
+ },
+ "uuid": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateCodeReq": {
+ "type": "object",
+ "properties": {
+ "admin": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string",
+ "example": "main"
+ },
+ "description": {
+ "type": "string"
+ },
+ "labels": {
+ "type": "string",
+ "example": ""
+ },
+ "license": {
+ "type": "string",
+ "example": "MIT"
+ },
+ "name": {
+ "type": "string",
+ "example": "model_name_1"
+ },
+ "namespace": {
+ "type": "string",
+ "example": "user_or_org_name"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateCollectionReq": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string",
+ "example": "collection1"
+ },
+ "namespace": {
+ "type": "string",
+ "example": "user_or_org_name"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "collection nick name"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "theme": {
+ "type": "string",
+ "example": "#fff000"
+ }
+ }
+ },
+ "types.CreateDatasetReq": {
+ "type": "object",
+ "properties": {
+ "admin": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string",
+ "example": "main"
+ },
+ "description": {
+ "type": "string"
+ },
+ "labels": {
+ "type": "string",
+ "example": ""
+ },
+ "license": {
+ "type": "string",
+ "example": "MIT"
+ },
+ "name": {
+ "type": "string",
+ "example": "model_name_1"
+ },
+ "namespace": {
+ "type": "string",
+ "example": "user_or_org_name"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ },
+ "type": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.CreateFileReq": {
+ "type": "object",
+ "properties": {
+ "branch": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "current_user": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "new_branch": {
+ "type": "string"
+ },
+ "original_content": {
+ "description": "Use for lfs file",
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "pointer": {
+ "$ref": "#/definitions/types.Pointer"
+ }
+ }
+ },
+ "types.CreateFileResp": {
+ "type": "object"
+ },
+ "types.CreateJWTReq": {
+ "type": "object",
+ "required": [
+ "uuid"
+ ],
+ "properties": {
+ "uuid": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateJWTResp": {
+ "type": "object",
+ "properties": {
+ "expire_at": {
+ "type": "string"
+ },
+ "token": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateLicenseReq": {
+ "type": "object",
+ "required": [
+ "company",
+ "edition",
+ "expire_time",
+ "max_user",
+ "product",
+ "start_time"
+ ],
+ "properties": {
+ "company": {
+ "type": "string"
+ },
+ "current_user": {
+ "type": "string"
+ },
+ "edition": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "expire_time": {
+ "type": "string"
+ },
+ "extra": {
+ "type": "string"
+ },
+ "max_user": {
+ "type": "integer",
+ "minimum": 1
+ },
+ "product": {
+ "type": "string"
+ },
+ "remark": {
+ "type": "string"
+ },
+ "start_time": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateMirrorParams": {
+ "type": "object",
+ "properties": {
+ "mirror_source_id": {
+ "type": "integer"
+ },
+ "password": {
+ "type": "string"
+ },
+ "source_url": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateMirrorRepoReq": {
+ "type": "object",
+ "required": [
+ "mirror_source_id",
+ "repo_type",
+ "source_name",
+ "source_namespace",
+ "source_url"
+ ],
+ "properties": {
+ "branch": {
+ "type": "string"
+ },
+ "current_user": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "license": {
+ "type": "string"
+ },
+ "mirror_source_id": {
+ "description": "source id for HF,github etc",
+ "type": "integer"
+ },
+ "repo_type": {
+ "description": "repo basic info",
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.RepositoryType"
+ }
+ ]
+ },
+ "source_name": {
+ "type": "string"
+ },
+ "source_namespace": {
+ "type": "string"
+ },
+ "source_url": {
+ "description": "mirror source info",
+ "type": "string"
+ },
+ "sync_lfs": {
+ "type": "boolean"
+ }
+ }
+ },
+ "types.CreateMirrorSourceReq": {
+ "type": "object",
+ "required": [
+ "source_name"
+ ],
+ "properties": {
+ "current_user": {
+ "type": "string"
+ },
+ "info_api_url": {
+ "type": "string"
+ },
+ "source_name": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateModelReq": {
+ "type": "object",
+ "properties": {
+ "admin": {
+ "type": "string"
+ },
+ "base_model": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string",
+ "example": "main"
+ },
+ "description": {
+ "type": "string"
+ },
+ "labels": {
+ "type": "string",
+ "example": ""
+ },
+ "license": {
+ "type": "string",
+ "example": "MIT"
+ },
+ "name": {
+ "type": "string",
+ "example": "model_name_1"
+ },
+ "namespace": {
+ "type": "string",
+ "example": "user_or_org_name"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateOrgReq": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string",
+ "example": "org description"
+ },
+ "homepage": {
+ "type": "string",
+ "example": "https://www.example.com"
+ },
+ "logo": {
+ "type": "string",
+ "example": "https://www.example.com/logo.png"
+ },
+ "name": {
+ "description": "Org unique identifier",
+ "type": "string",
+ "example": "org_name_1"
+ },
+ "nickname": {
+ "description": "Display name",
+ "type": "string",
+ "example": "org_display_name"
+ },
+ "org_type": {
+ "type": "string",
+ "example": "company or school etc"
+ },
+ "verified": {
+ "type": "boolean",
+ "example": false
+ }
+ }
+ },
+ "types.CreatePromptRepoReq": {
+ "type": "object",
+ "properties": {
+ "admin": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string",
+ "example": "main"
+ },
+ "description": {
+ "type": "string"
+ },
+ "labels": {
+ "type": "string",
+ "example": ""
+ },
+ "license": {
+ "type": "string",
+ "example": "MIT"
+ },
+ "name": {
+ "type": "string",
+ "example": "model_name_1"
+ },
+ "namespace": {
+ "type": "string",
+ "example": "user_or_org_name"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateSSHKeyRequest": {
+ "type": "object",
+ "properties": {
+ "content": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateSpaceReq": {
+ "type": "object",
+ "properties": {
+ "admin": {
+ "type": "string"
+ },
+ "cluster_id": {
+ "type": "string"
+ },
+ "cover_image_url": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string",
+ "example": "main"
+ },
+ "description": {
+ "type": "string"
+ },
+ "env": {
+ "type": "string"
+ },
+ "labels": {
+ "type": "string",
+ "example": ""
+ },
+ "license": {
+ "type": "string",
+ "example": "MIT"
+ },
+ "name": {
+ "type": "string",
+ "example": "model_name_1"
+ },
+ "namespace": {
+ "type": "string",
+ "example": "user_or_org_name"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "order_detail_id": {
+ "type": "integer"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "sdk": {
+ "type": "string",
+ "example": "1"
+ },
+ "sdk_version": {
+ "type": "string",
+ "example": "v0.1"
+ },
+ "secrets": {
+ "type": "string"
+ },
+ "template": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateSpaceResourceReq": {
+ "type": "object",
+ "required": [
+ "cluster_id",
+ "name",
+ "resources"
+ ],
+ "properties": {
+ "cluster_id": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "resources": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateSpaceSdkReq": {
+ "type": "object",
+ "required": [
+ "name",
+ "version"
+ ],
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateSyncClientSettingReq": {
+ "type": "object",
+ "required": [
+ "token"
+ ],
+ "properties": {
+ "concurrent_count": {
+ "type": "integer"
+ },
+ "max_bandwidth": {
+ "type": "integer"
+ },
+ "token": {
+ "type": "string"
+ }
+ }
+ },
+ "types.CreateUserResourceReq": {
+ "type": "object",
+ "properties": {
+ "order_details": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.AcctOrderDetailReq"
+ }
+ }
+ }
+ },
+ "types.CreateUserTokenRequest": {
+ "type": "object",
+ "properties": {
+ "application": {
+ "description": "default to csghub",
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.AccessTokenApp"
+ }
+ ]
+ },
+ "expired_at": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "permission": {
+ "description": "default to empty, means full permission",
+ "type": "string"
+ }
+ }
+ },
+ "types.Dataset": {
+ "type": "object",
+ "properties": {
+ "can_manage": {
+ "type": "boolean"
+ },
+ "can_write": {
+ "type": "boolean"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "downloads": {
+ "type": "integer"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "license": {
+ "type": "string"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "mirror_last_updated_at": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "$ref": "#/definitions/types.Namespace"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ },
+ "repository": {
+ "$ref": "#/definitions/types.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "sensitive_check_status": {
+ "type": "string"
+ },
+ "source": {
+ "$ref": "#/definitions/types.RepositorySource"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/types.User"
+ },
+ "user_likes": {
+ "type": "boolean"
+ }
+ }
+ },
+ "types.DatasetResp": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "downloads": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "types.DeleteUserTokenRequest": {
+ "type": "object",
+ "properties": {
+ "application": {
+ "description": "default to csghub",
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.AccessTokenApp"
+ }
+ ]
+ },
+ "expired_at": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "permission": {
+ "description": "default to empty, means full permission",
+ "type": "string"
+ }
+ }
+ },
+ "types.DeployUpdateReq": {
+ "type": "object",
+ "properties": {
+ "cluster_id": {
+ "type": "string"
+ },
+ "deploy_name": {
+ "type": "string"
+ },
+ "env": {
+ "type": "string"
+ },
+ "max_replica": {
+ "type": "integer",
+ "minimum": 1
+ },
+ "min_replica": {
+ "type": "integer",
+ "minimum": 1
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "revision": {
+ "type": "string"
+ },
+ "runtime_framework_id": {
+ "type": "integer"
+ },
+ "secure_level": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.EditOrgReq": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string",
+ "example": "org description"
+ },
+ "homepage": {
+ "type": "string",
+ "example": "https://www.example.com"
+ },
+ "logo": {
+ "type": "string",
+ "example": "https://www.example.com/logo.png"
+ },
+ "nickname": {
+ "description": "Display name",
+ "type": "string",
+ "example": "org display name"
+ },
+ "org_type": {
+ "type": "string",
+ "example": "company or school etc"
+ },
+ "verified": {
+ "type": "boolean",
+ "example": false
+ }
+ }
+ },
+ "types.EvaluationReq": {
+ "type": "object",
+ "properties": {
+ "datasets": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "model_id": {
+ "type": "string"
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "runtime_framework_id": {
+ "description": "ArgoWorkFlow framework",
+ "type": "integer"
+ },
+ "share_mode": {
+ "type": "boolean"
+ },
+ "task_desc": {
+ "type": "string"
+ },
+ "task_name": {
+ "type": "string"
+ }
+ }
+ },
+ "types.EvaluationRes": {
+ "type": "object",
+ "properties": {
+ "datasets": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTags"
+ }
+ },
+ "download_url": {
+ "type": "string"
+ },
+ "end_time": {
+ "type": "string"
+ },
+ "failures_url": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "image": {
+ "type": "string"
+ },
+ "reason": {
+ "type": "string"
+ },
+ "repo_ids": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "repo_type": {
+ "type": "string"
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "result_url": {
+ "type": "string"
+ },
+ "start_time": {
+ "type": "string"
+ },
+ "status": {
+ "type": "string"
+ },
+ "submit_time": {
+ "type": "string"
+ },
+ "task_desc": {
+ "type": "string"
+ },
+ "task_id": {
+ "type": "string"
+ },
+ "task_name": {
+ "type": "string"
+ },
+ "task_type": {
+ "$ref": "#/definitions/types.TaskType"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "types.Event": {
+ "type": "object",
+ "properties": {
+ "c_id": {
+ "type": "string",
+ "example": ""
+ },
+ "c_ip": {
+ "type": "string",
+ "example": ""
+ },
+ "ext": {
+ "description": "reserved for future use",
+ "type": "string",
+ "example": ""
+ },
+ "id": {
+ "type": "string",
+ "example": "space_card"
+ },
+ "m": {
+ "type": "string",
+ "example": "space"
+ },
+ "v": {
+ "type": "string",
+ "example": "1"
+ }
+ }
+ },
+ "types.File": {
+ "type": "object",
+ "properties": {
+ "commit": {
+ "$ref": "#/definitions/types.Commit"
+ },
+ "content": {
+ "type": "string"
+ },
+ "last_commit_sha": {
+ "type": "string"
+ },
+ "lfs": {
+ "type": "boolean"
+ },
+ "lfs_pointer_size": {
+ "type": "integer"
+ },
+ "lfs_relative_path": {
+ "description": "relative path in lfs storage",
+ "type": "string"
+ },
+ "mode": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "preview_code": {
+ "description": "whether file is previewable",
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.FilePreviewCode"
+ }
+ ]
+ },
+ "sha": {
+ "type": "string"
+ },
+ "size": {
+ "type": "integer"
+ },
+ "type": {
+ "type": "string"
+ },
+ "url": {
+ "description": "URL to browse the file",
+ "type": "string"
+ }
+ }
+ },
+ "types.FilePreviewCode": {
+ "type": "integer",
+ "enum": [
+ 0,
+ 1,
+ 2
+ ],
+ "x-enum-varnames": [
+ "FilePreviewCodeNormal",
+ "FilePreviewCodeTooLarge",
+ "FilePreviewCodeNotText"
+ ]
+ },
+ "types.ImportLicenseReq": {
+ "type": "object",
+ "required": [
+ "data"
+ ],
+ "properties": {
+ "data": {
+ "type": "string"
+ }
+ }
+ },
+ "types.InstanceRunReq": {
+ "type": "object",
+ "properties": {
+ "cluster_id": {
+ "type": "string"
+ },
+ "deploy_name": {
+ "type": "string"
+ },
+ "order_detail_id": {
+ "type": "integer"
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "revision": {
+ "type": "string"
+ },
+ "runtime_framework_id": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.LFSSyncProgressResp": {
+ "type": "object",
+ "properties": {
+ "oid": {
+ "type": "string"
+ },
+ "progress": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.ListByPathReq": {
+ "type": "object",
+ "properties": {
+ "paths": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "types.Member": {
+ "type": "object",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "last_login_at": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "role": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ },
+ "uuid": {
+ "type": "string"
+ }
+ }
+ },
+ "types.Mirror": {
+ "type": "object",
+ "properties": {
+ "access_token": {
+ "description": "source access token",
+ "type": "string"
+ },
+ "last_message": {
+ "type": "string"
+ },
+ "last_updated_at": {
+ "type": "string"
+ },
+ "local_repo_path": {
+ "type": "string"
+ },
+ "mirror_source": {
+ "$ref": "#/definitions/types.MirrorSource"
+ },
+ "progress": {
+ "type": "integer"
+ },
+ "push_access_token": {
+ "type": "string"
+ },
+ "push_url": {
+ "type": "string"
+ },
+ "push_username": {
+ "type": "string"
+ },
+ "repository": {
+ "$ref": "#/definitions/types.Repository"
+ },
+ "source_repo_path": {
+ "type": "string"
+ },
+ "source_url": {
+ "type": "string"
+ },
+ "status": {
+ "$ref": "#/definitions/types.MirrorTaskStatus"
+ },
+ "username": {
+ "description": "source user name",
+ "type": "string"
+ }
+ }
+ },
+ "types.MirrorPriority": {
+ "type": "integer",
+ "enum": [
+ 3,
+ 2,
+ 1
+ ],
+ "x-enum-varnames": [
+ "HighMirrorPriority",
+ "MediumMirrorPriority",
+ "LowMirrorPriority"
+ ]
+ },
+ "types.MirrorRepo": {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string"
+ },
+ "progress": {
+ "type": "integer"
+ },
+ "repo_type": {
+ "$ref": "#/definitions/types.RepositoryType"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ }
+ }
+ },
+ "types.MirrorSource": {
+ "type": "object",
+ "properties": {
+ "source_name": {
+ "type": "string"
+ }
+ }
+ },
+ "types.MirrorStatusCount": {
+ "type": "object",
+ "properties": {
+ "count": {
+ "type": "integer"
+ },
+ "status": {
+ "$ref": "#/definitions/types.MirrorTaskStatus"
+ }
+ }
+ },
+ "types.MirrorTaskStatus": {
+ "type": "string",
+ "enum": [
+ "waiting",
+ "running",
+ "repo_synced",
+ "finished",
+ "failed",
+ "incomplete"
+ ],
+ "x-enum-varnames": [
+ "MirrorWaiting",
+ "MirrorRunning",
+ "MirrorRepoSynced",
+ "MirrorFinished",
+ "MirrorFailed",
+ "MirrorIncomplete"
+ ]
+ },
+ "types.Model": {
+ "type": "object",
+ "properties": {
+ "base_model": {
+ "type": "string"
+ },
+ "can_manage": {
+ "type": "boolean"
+ },
+ "can_write": {
+ "type": "boolean"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "downloads": {
+ "type": "integer"
+ },
+ "enable_evaluation": {
+ "type": "boolean"
+ },
+ "enable_finetune": {
+ "type": "boolean"
+ },
+ "enable_inference": {
+ "type": "boolean"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "license": {
+ "type": "string"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "mirror_last_updated_at": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "$ref": "#/definitions/types.Namespace"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ },
+ "recom_op_weight": {
+ "type": "integer"
+ },
+ "repository": {
+ "$ref": "#/definitions/types.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "sensitive_check_status": {
+ "type": "string"
+ },
+ "source": {
+ "$ref": "#/definitions/types.RepositorySource"
+ },
+ "status": {
+ "description": "url to interact with the model",
+ "type": "string",
+ "example": "RUNNING"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/types.User"
+ },
+ "user_likes": {
+ "type": "boolean"
+ },
+ "widget_type": {
+ "description": "widget UI style: generation,chat",
+ "allOf": [
+ {
+ "$ref": "#/definitions/types.ModelWidgetType"
+ }
+ ],
+ "example": "generation"
+ }
+ }
+ },
+ "types.ModelPredictReq": {
+ "type": "object",
+ "properties": {
+ "current_user": {
+ "type": "string"
+ },
+ "input": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.ModelResp": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "downloads": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "types.ModelRunReq": {
+ "type": "object",
+ "properties": {
+ "cluster_id": {
+ "type": "string"
+ },
+ "deploy_name": {
+ "type": "string"
+ },
+ "env": {
+ "type": "string"
+ },
+ "max_replica": {
+ "type": "integer"
+ },
+ "min_replica": {
+ "type": "integer"
+ },
+ "order_detail_id": {
+ "type": "integer"
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "revision": {
+ "type": "string"
+ },
+ "runtime_framework_id": {
+ "type": "integer"
+ },
+ "secure_level": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.ModelWidgetType": {
+ "type": "string",
+ "enum": [
+ "generation",
+ "chat"
+ ],
+ "x-enum-varnames": [
+ "ModelWidgetTypeGeneration",
+ "ModelWidgetTypeChat"
+ ]
+ },
+ "types.Namespace": {
+ "type": "object",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "type": {
+ "description": "namespace types like 'user' for normal user, and 'school', 'company' for orgs etc.",
+ "type": "string"
+ }
+ }
+ },
+ "types.Organization": {
+ "type": "object",
+ "properties": {
+ "homepage": {
+ "type": "string"
+ },
+ "logo": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "org_type": {
+ "type": "string"
+ },
+ "path": {
+ "description": "unique name of the organization",
+ "type": "string"
+ },
+ "user_id": {
+ "type": "integer"
+ },
+ "verified": {
+ "type": "boolean"
+ }
+ }
+ },
+ "types.PayMode": {
+ "type": "string",
+ "enum": [
+ "free",
+ "minute",
+ "month",
+ "year"
+ ],
+ "x-enum-varnames": [
+ "PayModeFree",
+ "PayModeMinute",
+ "PayModeMonth",
+ "PayModeYear"
+ ]
+ },
+ "types.Pointer": {
+ "type": "object",
+ "properties": {
+ "download_url": {
+ "type": "string"
+ },
+ "oid": {
+ "type": "string"
+ },
+ "size": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.PromptRes": {
+ "type": "object",
+ "properties": {
+ "can_manage": {
+ "type": "boolean"
+ },
+ "can_write": {
+ "type": "boolean"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "downloads": {
+ "type": "integer"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "license": {
+ "type": "string"
+ },
+ "likes": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "namespace": {
+ "$ref": "#/definitions/types.Namespace"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "path": {
+ "type": "string"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "readme": {
+ "type": "string"
+ },
+ "repository": {
+ "$ref": "#/definitions/types.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "sensitive_check_status": {
+ "type": "string"
+ },
+ "source": {
+ "$ref": "#/definitions/types.RepositorySource"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/types.User"
+ },
+ "user_likes": {
+ "type": "boolean"
+ }
+ }
+ },
+ "types.RECHARGE_REQ": {
+ "type": "object",
+ "properties": {
+ "op_uid": {
+ "type": "string"
+ },
+ "scene": {
+ "type": "integer"
+ },
+ "value": {
+ "type": "number",
+ "minimum": 1
+ }
+ }
+ },
+ "types.RechargeStatusResp": {
+ "type": "object",
+ "properties": {
+ "rechargeSucceeded": {
+ "type": "boolean"
+ },
+ "rechargeUUID": {
+ "type": "string"
+ }
+ }
+ },
+ "types.RelationDataset": {
+ "type": "object",
+ "properties": {
+ "dataset": {
+ "type": "string"
+ }
+ }
+ },
+ "types.RelationDatasets": {
+ "type": "object",
+ "properties": {
+ "datasets": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "types.RelationModel": {
+ "type": "object",
+ "properties": {
+ "model": {
+ "type": "string"
+ }
+ }
+ },
+ "types.RelationModels": {
+ "type": "object",
+ "properties": {
+ "models": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "types.Relations": {
+ "type": "object",
+ "properties": {
+ "codes": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Code"
+ }
+ },
+ "datasets": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Dataset"
+ }
+ },
+ "models": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Model"
+ }
+ },
+ "prompts": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.PromptRes"
+ }
+ },
+ "spaces": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Space"
+ }
+ }
+ }
+ },
+ "types.RepoBranchCommit": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "string"
+ }
+ }
+ },
+ "types.RepoTag": {
+ "type": "object",
+ "properties": {
+ "built_in": {
+ "type": "boolean"
+ },
+ "category": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "group": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "show_name": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ }
+ }
+ },
+ "types.RepoTags": {
+ "type": "object",
+ "properties": {
+ "repo_id": {
+ "type": "string"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ }
+ }
+ },
+ "types.Repository": {
+ "type": "object",
+ "properties": {
+ "http_clone_url": {
+ "type": "string"
+ },
+ "ssh_clone_url": {
+ "type": "string"
+ }
+ }
+ },
+ "types.RepositorySource": {
+ "type": "string",
+ "enum": [
+ "opencsg",
+ "local",
+ "huggingface"
+ ],
+ "x-enum-varnames": [
+ "OpenCSGSource",
+ "LocalSource",
+ "HuggingfaceSource"
+ ]
+ },
+ "types.RepositorySyncStatus": {
+ "type": "string",
+ "enum": [
+ "pending",
+ "inprogress",
+ "failed",
+ "completed"
+ ],
+ "x-enum-varnames": [
+ "SyncStatusPending",
+ "SyncStatusInProgress",
+ "SyncStatusFailed",
+ "SyncStatusCompleted"
+ ]
+ },
+ "types.RepositoryType": {
+ "type": "string",
+ "enum": [
+ "model",
+ "dataset",
+ "space",
+ "code",
+ "prompt",
+ ""
+ ],
+ "x-enum-varnames": [
+ "ModelRepo",
+ "DatasetRepo",
+ "SpaceRepo",
+ "CodeRepo",
+ "PromptRepo",
+ "UnknownRepo"
+ ]
+ },
+ "types.ResourceType": {
+ "type": "string",
+ "enum": [
+ "cpu",
+ "gpu",
+ "npu"
+ ],
+ "x-enum-varnames": [
+ "ResourceTypeCPU",
+ "ResourceTypeGPU",
+ "ResourceTypeNPU"
+ ]
+ },
+ "types.Response": {
+ "type": "object",
+ "properties": {
+ "data": {},
+ "msg": {
+ "type": "string"
+ }
+ }
+ },
+ "types.ResponseWithTotal": {
+ "type": "object",
+ "properties": {
+ "data": {},
+ "msg": {
+ "type": "string"
+ },
+ "total": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.RuntimeArchitecture": {
+ "type": "object",
+ "properties": {
+ "architectures": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "types.RuntimeFramework": {
+ "type": "object",
+ "properties": {
+ "container_port": {
+ "type": "integer"
+ },
+ "enabled": {
+ "type": "integer"
+ },
+ "frame_cpu_image": {
+ "type": "string"
+ },
+ "frame_image": {
+ "type": "string"
+ },
+ "frame_name": {
+ "type": "string"
+ },
+ "frame_npu_image": {
+ "type": "string"
+ },
+ "frame_version": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "type": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.RuntimeFrameworkModels": {
+ "type": "object",
+ "properties": {
+ "models": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "types.RuntimeFrameworkReq": {
+ "type": "object",
+ "properties": {
+ "container_port": {
+ "type": "integer"
+ },
+ "enabled": {
+ "type": "integer"
+ },
+ "frame_cpu_image": {
+ "type": "string"
+ },
+ "frame_image": {
+ "type": "string"
+ },
+ "frame_name": {
+ "type": "string"
+ },
+ "frame_version": {
+ "type": "string"
+ },
+ "type": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.SensitiveCheckStatus": {
+ "type": "integer",
+ "enum": [
+ -1,
+ 0,
+ 1,
+ 2,
+ 3
+ ],
+ "x-enum-comments": {
+ "SensitiveCheckException": "error happen",
+ "SensitiveCheckFail": "sensitive content detected",
+ "SensitiveCheckPass": "pass",
+ "SensitiveCheckPending": "default",
+ "SensitiveCheckSkip": "skip"
+ },
+ "x-enum-varnames": [
+ "SensitiveCheckFail",
+ "SensitiveCheckPending",
+ "SensitiveCheckPass",
+ "SensitiveCheckSkip",
+ "SensitiveCheckException"
+ ]
+ },
+ "types.Space": {
+ "type": "object",
+ "properties": {
+ "can_manage": {
+ "type": "boolean"
+ },
+ "can_write": {
+ "type": "boolean"
+ },
+ "cover_image_url": {
+ "type": "string"
+ },
+ "created_at": {
+ "type": "string"
+ },
+ "default_branch": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string",
+ "example": ""
+ },
+ "endpoint": {
+ "description": "the serving endpoint url",
+ "type": "string",
+ "example": "https://localhost/spaces/myname/myspace"
+ },
+ "env": {
+ "type": "string"
+ },
+ "hardware": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "license": {
+ "type": "string",
+ "example": "MIT"
+ },
+ "like_count": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string",
+ "example": "space_name_1"
+ },
+ "namespace": {
+ "$ref": "#/definitions/types.Namespace"
+ },
+ "nickname": {
+ "type": "string",
+ "example": ""
+ },
+ "path": {
+ "type": "string",
+ "example": "user_or_org_name/space_name_1"
+ },
+ "private": {
+ "type": "boolean"
+ },
+ "repository": {
+ "$ref": "#/definitions/types.Repository"
+ },
+ "repository_id": {
+ "type": "integer"
+ },
+ "sdk": {
+ "description": "like gradio,steamlit etc",
+ "type": "string",
+ "example": "1"
+ },
+ "sdk_version": {
+ "type": "string",
+ "example": "v0.1"
+ },
+ "secrets": {
+ "type": "string"
+ },
+ "sensitive_check_status": {
+ "type": "string"
+ },
+ "sku": {
+ "type": "string"
+ },
+ "source": {
+ "$ref": "#/definitions/types.RepositorySource"
+ },
+ "status": {
+ "description": "deploying, running, failed",
+ "type": "string"
+ },
+ "svc_name": {
+ "type": "string"
+ },
+ "sync_status": {
+ "$ref": "#/definitions/types.RepositorySyncStatus"
+ },
+ "tags": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.RepoTag"
+ }
+ },
+ "template": {
+ "type": "string"
+ },
+ "updated_at": {
+ "type": "string"
+ },
+ "user": {
+ "$ref": "#/definitions/types.User"
+ },
+ "user_likes": {
+ "type": "boolean"
+ },
+ "username": {
+ "type": "string",
+ "example": "creator_user_name"
+ }
+ }
+ },
+ "types.SpaceResource": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "integer"
+ },
+ "is_available": {
+ "type": "boolean"
+ },
+ "is_reserved": {
+ "type": "boolean"
+ },
+ "name": {
+ "type": "string"
+ },
+ "order_detail_id": {
+ "type": "integer"
+ },
+ "pay_mode": {
+ "$ref": "#/definitions/types.PayMode"
+ },
+ "price": {
+ "type": "number"
+ },
+ "resources": {
+ "type": "string"
+ },
+ "type": {
+ "$ref": "#/definitions/types.ResourceType"
+ }
+ }
+ },
+ "types.SpaceSdk": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.SyncVersion": {
+ "type": "object",
+ "properties": {
+ "change_log": {
+ "type": "string"
+ },
+ "last_modify_time": {
+ "type": "string"
+ },
+ "repo_path": {
+ "type": "string"
+ },
+ "repo_type": {
+ "$ref": "#/definitions/types.RepositoryType"
+ },
+ "source_id": {
+ "description": "0: opencsg, 1: HF",
+ "type": "integer"
+ },
+ "version": {
+ "type": "integer"
+ }
+ }
+ },
+ "types.SyncVersionResponse": {
+ "type": "object",
+ "properties": {
+ "data": {
+ "type": "object",
+ "properties": {
+ "has_more": {
+ "type": "boolean"
+ },
+ "versions": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.SyncVersion"
+ }
+ }
+ }
+ },
+ "msg": {
+ "type": "string"
+ }
+ }
+ },
+ "types.TaskType": {
+ "type": "string",
+ "enum": [
+ "evaluation",
+ "training",
+ "comparison",
+ "leaderboard"
+ ],
+ "x-enum-varnames": [
+ "TaskTypeEvaluation",
+ "TaskTypeTraining",
+ "TaskTypeComparison",
+ "TaskTypeLeaderBoard"
+ ]
+ },
+ "types.UpdateCodeReq": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean",
+ "example": false
+ }
+ }
+ },
+ "types.UpdateCollectionReposReq": {
+ "type": "object",
+ "properties": {
+ "repo_ids": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ }
+ }
+ },
+ "types.UpdateDatasetReq": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean",
+ "example": false
+ }
+ }
+ },
+ "types.UpdateFileReq": {
+ "type": "object",
+ "properties": {
+ "branch": {
+ "type": "string"
+ },
+ "content": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "new_branch": {
+ "type": "string"
+ },
+ "origin_path": {
+ "type": "string"
+ },
+ "original_content": {
+ "description": "Use for lfs file",
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "pointer": {
+ "$ref": "#/definitions/types.Pointer"
+ },
+ "repoType": {
+ "$ref": "#/definitions/types.RepositoryType"
+ },
+ "sha": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateFileResp": {
+ "type": "object"
+ },
+ "types.UpdateLicenseReq": {
+ "type": "object",
+ "properties": {
+ "company": {
+ "type": "string"
+ },
+ "edition": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "expire_time": {
+ "type": "string"
+ },
+ "extra": {
+ "type": "string"
+ },
+ "max_user": {
+ "type": "integer"
+ },
+ "product": {
+ "type": "string"
+ },
+ "remark": {
+ "type": "string"
+ },
+ "start_time": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateMirrorParams": {
+ "type": "object",
+ "properties": {
+ "mirror_source_id": {
+ "type": "integer"
+ },
+ "password": {
+ "type": "string"
+ },
+ "source_url": {
+ "type": "string"
+ },
+ "username": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateMirrorSourceReq": {
+ "type": "object",
+ "required": [
+ "source_name"
+ ],
+ "properties": {
+ "current_user": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "info_api_url": {
+ "type": "string"
+ },
+ "source_name": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateModelReq": {
+ "type": "object",
+ "properties": {
+ "base_model": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean",
+ "example": false
+ }
+ }
+ },
+ "types.UpdatePromptRepoReq": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "private": {
+ "type": "boolean",
+ "example": false
+ }
+ }
+ },
+ "types.UpdateSpaceReq": {
+ "type": "object",
+ "properties": {
+ "cover_image_url": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string"
+ },
+ "env": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string",
+ "example": "model display name"
+ },
+ "order_detail_id": {
+ "type": "integer"
+ },
+ "private": {
+ "type": "boolean",
+ "example": false
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "sdk": {
+ "type": "string",
+ "example": "1"
+ },
+ "sdk_version": {
+ "type": "string",
+ "example": "v0.1"
+ },
+ "secrets": {
+ "type": "string"
+ },
+ "template": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateSpaceResourceReq": {
+ "type": "object",
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "resources": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateSpaceSdkReq": {
+ "type": "object",
+ "properties": {
+ "id": {
+ "type": "integer"
+ },
+ "name": {
+ "type": "string"
+ },
+ "version": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UpdateUserRequest": {
+ "type": "object",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "bio": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "homepage": {
+ "type": "string"
+ },
+ "name": {
+ "description": "Display name of the user",
+ "type": "string"
+ },
+ "new_username": {
+ "description": "if use want to change username, this should be the only field to send in request body",
+ "type": "string"
+ },
+ "phone": {
+ "type": "string"
+ },
+ "roles": {
+ "description": "should be updated by admin",
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "example": [
+ "[super_user",
+ " admin",
+ " personal_user]"
+ ]
+ },
+ "uuid": {
+ "type": "string"
+ }
+ }
+ },
+ "types.User": {
+ "type": "object",
+ "properties": {
+ "avatar": {
+ "type": "string"
+ },
+ "bio": {
+ "type": "string"
+ },
+ "can_change_username": {
+ "type": "boolean"
+ },
+ "email": {
+ "type": "string"
+ },
+ "homepage": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "last_login_at": {
+ "type": "string"
+ },
+ "nickname": {
+ "type": "string"
+ },
+ "orgs": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/types.Organization"
+ }
+ },
+ "phone": {
+ "type": "string"
+ },
+ "roles": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "username": {
+ "type": "string"
+ },
+ "uuid": {
+ "type": "string"
+ }
+ }
+ },
+ "types.UserResourcesResp": {
+ "type": "object",
+ "properties": {
+ "created_at": {
+ "type": "string"
+ },
+ "deploy_id": {
+ "type": "integer"
+ },
+ "deploy_name": {
+ "type": "string"
+ },
+ "deploy_type": {
+ "type": "integer"
+ },
+ "end_time": {
+ "type": "string"
+ },
+ "id": {
+ "type": "integer"
+ },
+ "order_detail_id": {
+ "type": "integer"
+ },
+ "order_id": {
+ "type": "string"
+ },
+ "pay_mode": {
+ "type": "string"
+ },
+ "price": {
+ "type": "number"
+ },
+ "repo_path": {
+ "type": "string"
+ },
+ "resource": {
+ "type": "string"
+ },
+ "resource_id": {
+ "type": "integer"
+ },
+ "resource_type": {
+ "type": "string"
+ },
+ "start_time": {
+ "type": "string"
+ },
+ "xpu_num": {
+ "type": "integer"
+ }
+ }
+ }
+ },
+ "securityDefinitions": {
+ "ApiKey": {
+ "description": "Bearer token",
+ "type": "apiKey",
+ "name": "Authorization",
+ "in": "header"
+ }
+ }
+}`
+
+// SwaggerInfo holds exported Swagger Info so clients can modify it
+var SwaggerInfo = &swag.Spec{
+ Version: "",
+ Host: "",
+ BasePath: "",
+ Schemes: []string{},
+ Title: "",
+ Description: "",
+ InfoInstanceName: "swagger",
+ SwaggerTemplate: docTemplate,
+ LeftDelim: "{{",
+ RightDelim: "}}",
+}
+
+func init() {
+ swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
+}
+
+
+
package cache
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strconv"
+
+ "github.com/aliyun/aliyun-oss-go-sdk/oss"
+ "opencsg.com/csghub-server/builder/store/cache"
+ "opencsg.com/csghub-server/common/config"
+)
+
+const (
+ OssMultipartUploadKeyPrefix = "oss-multipart-upload"
+ LfsSyncKeyPrefix = "lfs-sync"
+ LfsSyncProgressKeyPrefix = "lfs-sync-progress"
+)
+
+type Cache interface {
+ CacheImur(ctx context.Context, repoPath, oid string, o *OssMultipartUploadResult) error
+ GetImur(ctx context.Context, repoPath, oid string) (*OssMultipartUploadResult, error)
+ DeleteImur(ctx context.Context, repoPath, oid string) error
+
+ CacheLfsSyncAddPart(ctx context.Context, repoPath, oid string, partNumber int) error
+ IsLfsPartSynced(ctx context.Context, repoPath, oid string, partNumber int) (bool, error)
+ LfsPartSyncedCount(ctx context.Context, repoPath, oid string) (int64, error)
+ DeleteLfsPartCache(ctx context.Context, repoPath, oid string) error
+
+ CacheLfsSyncFileProgress(ctx context.Context, repoPath, oid string, progress int) error
+ GetLfsSyncFileProgress(ctx context.Context, repoPath, oid string) (int, error)
+}
+
+type cacheImpl struct {
+ redis *cache.Cache
+}
+
+func NewCache(ctx context.Context, config *config.Config) (Cache, error) {
+ redis, err := cache.NewCache(ctx, cache.RedisConfig{
+ Addr: config.Redis.Endpoint,
+ Username: config.Redis.User,
+ Password: config.Redis.Password,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("initializing redis: %w", err)
+ }
+ return &cacheImpl{
+ redis: redis,
+ }, nil
+}
+
+type OssMultipartUploadResult struct {
+ Imur oss.InitiateMultipartUploadResult
+}
+
+func (o *OssMultipartUploadResult) MarshalBinary() ([]byte, error) {
+ return json.Marshal(o)
+}
+
+func (o *OssMultipartUploadResult) UnmarshalBinary(data []byte) error {
+ return json.Unmarshal(data, o)
+}
+
+func (c *cacheImpl) CacheImur(ctx context.Context, repoPath, oid string, o *OssMultipartUploadResult) error {
+ key := imurKeyCacheKey(repoPath, oid)
+ data, err := o.MarshalBinary()
+ if err != nil {
+ return fmt.Errorf("failed to marshal OssMultipartUploadResult: %w", err)
+ }
+ return c.redis.Set(ctx, key, string(data))
+}
+
+func (c *cacheImpl) GetImur(ctx context.Context, repoPath, oid string) (*OssMultipartUploadResult, error) {
+ key := imurKeyCacheKey(repoPath, oid)
+ o := new(OssMultipartUploadResult)
+ data, err := c.redis.Get(ctx, key)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get imur: %w", err)
+ }
+ err = o.UnmarshalBinary([]byte(data))
+ if err != nil {
+ return nil, fmt.Errorf("failed to get imur: %w", err)
+ }
+ return o, nil
+}
+
+func (c *cacheImpl) DeleteImur(ctx context.Context, repoPath, oid string) error {
+ key := imurKeyCacheKey(repoPath, oid)
+ return c.redis.Del(ctx, key)
+}
+
+func (c *cacheImpl) CacheLfsSyncAddPart(ctx context.Context, repoPath, oid string, partNumber int) error {
+ key := lfsPartCacheKey(repoPath, oid)
+ err := c.redis.SAdd(ctx, key, partNumber)
+ if err != nil {
+ return fmt.Errorf("failed to add lfs part number to set: %w", err)
+ }
+ return nil
+}
+
+func (c *cacheImpl) IsLfsPartSynced(ctx context.Context, repoPath, oid string, partNumber int) (bool, error) {
+ key := lfsPartCacheKey(repoPath, oid)
+ return c.redis.SIsMember(ctx, key, partNumber)
+}
+
+func (c *cacheImpl) LfsPartSyncedCount(ctx context.Context, repoPath, oid string) (int64, error) {
+ key := lfsPartCacheKey(repoPath, oid)
+ return c.redis.SCard(ctx, key)
+}
+
+func (c *cacheImpl) DeleteLfsPartCache(ctx context.Context, repoPath, oid string) error {
+ key := lfsPartCacheKey(repoPath, oid)
+ return c.redis.Del(ctx, key)
+}
+
+func (c *cacheImpl) CacheLfsSyncFileProgress(ctx context.Context, repoPath, oid string, progress int) error {
+ key := lfsProgressCacheKey(repoPath, oid)
+ strProgress := strconv.Itoa(progress)
+ err := c.redis.Set(ctx, key, strProgress)
+ if err != nil {
+ return fmt.Errorf("failed to set lfs part number to set: %w", err)
+ }
+ return nil
+}
+
+func (c *cacheImpl) GetLfsSyncFileProgress(ctx context.Context, repoPath, oid string) (int, error) {
+ key := lfsProgressCacheKey(repoPath, oid)
+ strProgress, err := c.redis.Get(ctx, key)
+ if err != nil {
+ return 0, fmt.Errorf("failed to get lfs part number: %w", err)
+ }
+ return strconv.Atoi(strProgress)
+}
+func lfsProgressCacheKey(repoPath, oid string) string {
+ return fmt.Sprintf("%s-%s-%s", LfsSyncProgressKeyPrefix, repoPath, oid)
+}
+func lfsPartCacheKey(repoPath, oid string) string {
+ return fmt.Sprintf("%s-%s-%s", LfsSyncKeyPrefix, repoPath, oid)
+}
+
+func imurKeyCacheKey(repoPath, oid string) string {
+ return fmt.Sprintf("%s-%s-%s", OssMultipartUploadKeyPrefix, repoPath, oid)
+}
+
+
+
package mirror
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/mirror/lfssyncer"
+)
+
+type LFSSyncWorker interface {
+ Run()
+ SyncLfs(ctx context.Context, workerID int, mirror *database.Mirror) error
+}
+
+func NewLFSSyncWorker(config *config.Config, numWorkers int) (LFSSyncWorker, error) {
+ if config.Mirror.Remote {
+ return lfssyncer.NewAliyunOssLfSSyncWorker(config, numWorkers)
+ }
+
+ return lfssyncer.NewMinioLFSSyncWorker(config, numWorkers)
+}
+
+
+
package lfssyncer
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "math"
+ "net/http"
+ "net/url"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/aliyun/aliyun-oss-go-sdk/oss"
+ "golang.org/x/sync/errgroup"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mirror/cache"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+type repoPathKey string
+
+type workerIDKey string
+
+var (
+ wk workerIDKey = "workerID"
+ rk repoPathKey = "repoPath"
+ maxRetries int = 3
+)
+
+type AliyunOssLfSSyncWorker struct {
+ mq queue.PriorityQueue
+ tasks chan queue.MirrorTask
+ wg sync.WaitGroup
+ mirrorStore database.MirrorStore
+ lfsMetaObjectStore database.LfsMetaObjectStore
+ ossClient *oss.Client
+ bucket *oss.Bucket
+ config *config.Config
+ syncCache cache.Cache
+ numWorkers int
+ mu sync.Mutex
+}
+
+func NewAliyunOssLfSSyncWorker(config *config.Config, numWorkers int) (*AliyunOssLfSSyncWorker, error) {
+ var err error
+ w := &AliyunOssLfSSyncWorker{}
+ w.numWorkers = numWorkers
+ w.ossClient, err = oss.New(
+ config.S3.Endpoint,
+ config.S3.AccessKeyID,
+ config.S3.AccessKeySecret,
+ )
+ if err != nil {
+ newError := fmt.Errorf("fail to init aliyun oss client for code,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ w.bucket, err = w.ossClient.Bucket(config.S3.Bucket)
+ if err != nil {
+ newError := fmt.Errorf("fail to get aliyun oss bucket for code,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ w.mirrorStore = database.NewMirrorStore()
+ w.lfsMetaObjectStore = database.NewLfsMetaObjectStore()
+ w.config = config
+ mq, err := queue.GetPriorityQueueInstance()
+ if err != nil {
+ return nil, fmt.Errorf("fail to get priority queue: %w", err)
+ }
+ w.mq = mq
+ cache, err := cache.NewCache(context.Background(), config)
+ if err != nil {
+ return nil, fmt.Errorf("initializing redis: %w", err)
+ }
+ w.syncCache = cache
+ w.tasks = make(chan queue.MirrorTask)
+ return w, nil
+}
+
+func (w *AliyunOssLfSSyncWorker) Run() {
+ for i := 1; i <= w.numWorkers; i++ {
+ w.wg.Add(1)
+ go w.worker(i)
+ }
+ go w.dispatcher()
+ w.wg.Wait()
+}
+
+func (w *AliyunOssLfSSyncWorker) dispatcher() {
+ for {
+ task := w.mq.PopLfsMirror()
+ if task != nil {
+ w.tasks <- *task
+ }
+ }
+}
+
+func (w *AliyunOssLfSSyncWorker) worker(id int) {
+ defer w.wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ w.wg.Add(1)
+ go w.worker(id)
+ slog.Info("worker ecovered from panic ", slog.Int("workerId", id))
+ }
+ }()
+ slog.Info("worker start", slog.Int("workerId", id))
+ for {
+ task := <-w.tasks
+ ctx := context.Background()
+ mirror, err := w.mirrorStore.FindByID(ctx, task.MirrorID)
+ if err != nil {
+ slog.Error("fail to get mirror", slog.Int("workerId", id), slog.String("error", err.Error()))
+ continue
+ }
+ err = w.SyncLfs(ctx, id, mirror)
+ if err != nil {
+ slog.Error("fail to sync lfs", slog.Int("workerId", id), slog.String("error", err.Error()))
+ continue
+ }
+ }
+}
+
+func (w *AliyunOssLfSSyncWorker) SyncLfs(ctx context.Context, workerId int, mirror *database.Mirror) error {
+ var pointers []*types.Pointer
+ ctx = context.WithValue(ctx, wk, workerId)
+ repoPath := fmt.Sprintf("%ss/%s", mirror.Repository.RepositoryType, mirror.Repository.Path)
+ ctx = context.WithValue(ctx, rk, repoPath)
+ slog.Info("start sync lfs", slog.Int("workerId", workerId), slog.String("repoPath", repoPath))
+ // Query all lfsMetaObjects to generate the &types.Pointer slice
+ lfsMetaObjects, err := w.lfsMetaObjectStore.FindByRepoID(ctx, mirror.Repository.ID)
+ if err != nil {
+ slog.Error("fail to get lfs meta objects", slog.Int("workerId", workerId), slog.String("error", err.Error()))
+ return fmt.Errorf("fail to get lfs meta objects: %w", err)
+ }
+ for _, lfsMetaObject := range lfsMetaObjects {
+ pointers = append(pointers, &types.Pointer{
+ Oid: lfsMetaObject.Oid,
+ Size: lfsMetaObject.Size,
+ })
+ }
+
+ err = w.DownloadAndUploadLFSFiles(ctx, mirror, pointers)
+ if err != nil {
+ return fmt.Errorf("fail to download and upload LFS files: %w", err)
+ }
+
+ return nil
+}
+
+func (w *AliyunOssLfSSyncWorker) GetLFSDownloadURL(ctx context.Context, mirror *database.Mirror, pointer *types.Pointer) (*types.Pointer, error) {
+ var (
+ resPointers []*types.Pointer
+ lfsAPIURL string
+ )
+ requestPayload := types.LFSBatchRequest{
+ Operation: "download",
+ }
+
+ requestPayload.Objects = append(requestPayload.Objects, types.LFSBatchObject{
+ Oid: pointer.Oid,
+ Size: pointer.Size,
+ })
+ requestPayload.HashAlog = "sha256"
+ requestPayload.Transfers = []string{"lfs-standalone-file", "basic", "bash"}
+ requestPayload.Ref = types.LFSBatchObjectRef{
+ Name: fmt.Sprintf("refs/heads/%s", mirror.Repository.DefaultBranch),
+ }
+ if strings.HasSuffix(mirror.SourceUrl, ".git") {
+ lfsAPIURL = mirror.SourceUrl + "/info/lfs/objects/batch"
+ } else {
+ lfsAPIURL = mirror.SourceUrl + ".git/info/lfs/objects/batch"
+ }
+
+ payload, err := json.Marshal(requestPayload)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal request payload: %v", err)
+ }
+
+ req, err := http.NewRequest("POST", lfsAPIURL, bytes.NewReader(payload))
+ if err != nil {
+ return nil, fmt.Errorf("failed to create LFS batch request: %v", err)
+ }
+
+ parsedURL, err := url.Parse(lfsAPIURL)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse LFS API URL: %v", err)
+ }
+
+ req.Header.Set("Host", parsedURL.Host)
+ req.Header.Set("Accept", "application/vnd.git-lfs+json")
+ req.Header.Set("Content-Type", "application/vnd.git-lfs+json; charset=utf-8")
+ req.Header.Set("User-Agent", "git-lfs/3.5.1")
+
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to send LFS batch request: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("failed to get LFS download URL, status code: %d", resp.StatusCode)
+ }
+
+ var batchResponse types.LFSBatchResponse
+ err = json.NewDecoder(resp.Body).Decode(&batchResponse)
+ if err != nil {
+ return nil, fmt.Errorf("failed to decode LFS batch response: %v", err)
+ }
+
+ if len(batchResponse.Objects) == 0 {
+ return nil, fmt.Errorf("no objects found in LFS batch response")
+ }
+ for _, object := range batchResponse.Objects {
+ resPointers = append(resPointers, &types.Pointer{
+ Oid: object.Oid,
+ Size: object.Size,
+ DownloadURL: object.Actions.Download.Href,
+ })
+ }
+
+ return resPointers[0], nil
+}
+
+func (w *AliyunOssLfSSyncWorker) DownloadAndUploadLFSFiles(ctx context.Context, mirror *database.Mirror, pointers []*types.Pointer) error {
+ var (
+ finishedLFSFileCount int
+ exists bool
+ err error
+ )
+ repoPath := ctx.Value(rk).(string)
+ lfsFilesCount := len(pointers)
+ for _, pointer := range pointers {
+ // Generate the object key for the LFS file
+ objectKey := filepath.Join("lfs", pointer.RelativePath())
+ exists, err = w.bucket.IsObjectExist(objectKey)
+ if err != nil {
+ slog.Error("failed to check if LFS file exists", slog.Any("error", err))
+ continue
+ }
+ if !exists {
+ err = w.DownloadAndUploadLFSFile(ctx, mirror, pointer)
+ if err != nil {
+ lfsMetaObject := database.LfsMetaObject{
+ Size: pointer.Size,
+ Oid: pointer.Oid,
+ RepositoryID: mirror.Repository.ID,
+ Existing: false,
+ }
+ _, err = w.lfsMetaObjectStore.UpdateOrCreate(ctx, lfsMetaObject)
+ if err != nil {
+ slog.Error("failed to update or create LFS meta object", slog.Any("error", err))
+ return fmt.Errorf("failed to update or create LFS meta object: %w", err)
+ }
+ continue
+ } else {
+ exists = true
+ }
+ }
+ // If exists, set the progress of lfs file to 100
+ err = w.syncCache.CacheLfsSyncFileProgress(ctx, repoPath, pointer.Oid, 100)
+ if err != nil {
+ slog.Error("failed to cache lfs upload progress", slog.Any("oid", pointer.Oid), slog.Any("error", err))
+ return fmt.Errorf("failed to cache lfs upload progress: %w", err)
+ }
+
+ lfsMetaObject := database.LfsMetaObject{
+ Size: pointer.Size,
+ Oid: pointer.Oid,
+ RepositoryID: mirror.Repository.ID,
+ Existing: exists,
+ }
+ _, err = w.lfsMetaObjectStore.UpdateOrCreate(ctx, lfsMetaObject)
+ if err != nil {
+ slog.Error("failed to update or create LFS meta object", slog.Any("error", err))
+ return fmt.Errorf("failed to update or create LFS meta object: %w", err)
+ }
+ finishedLFSFileCount += 1
+ mirror.Progress = int8(finishedLFSFileCount * 100 / lfsFilesCount)
+ err = w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror progress: %w", err)
+ }
+ }
+ mirror.Status = types.MirrorFinished
+ err = w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror status: %w", err)
+ }
+ return nil
+}
+
+func (w *AliyunOssLfSSyncWorker) DownloadAndUploadLFSFile(ctx context.Context, mirror *database.Mirror, pointer *types.Pointer) error {
+ var imur oss.InitiateMultipartUploadResult
+ workerId := ctx.Value(wk).(int)
+ repoPath := ctx.Value(rk).(string)
+ pointer, err := w.GetLFSDownloadURL(ctx, mirror, pointer)
+ if err != nil {
+ slog.Error("fail to get LFS download URL", slog.Int("workerId", workerId), slog.String("error", err.Error()), slog.Any("pointer", pointer), slog.Any("repo_path", repoPath))
+ return fmt.Errorf("fail to get LFS download URL: %w", err)
+ }
+ partSize := int64(w.config.Mirror.PartSize * 1024 * 1024)
+ objectKey := filepath.Join("lfs", pointer.RelativePath())
+ // Use cache.OssMultipartUploadResult to wrap the oss.InitiateMultipartUploadResult struct
+ // Check if the OssMultipartUploadResult exists in cache
+ imurWrapper, _ := w.syncCache.GetImur(ctx, repoPath, pointer.Oid)
+ if imurWrapper == nil {
+ // If the OssMultipartUploadResult does not exist in cache, initiate a multipart upload and store it in cache
+ imur, err = w.bucket.InitiateMultipartUpload(objectKey)
+ if err == nil {
+ err = w.syncCache.CacheImur(ctx, repoPath, pointer.Oid, &cache.OssMultipartUploadResult{
+ Imur: imur,
+ })
+ } else {
+ slog.Error("aliyun InitiateMultipartUpload failed", slog.Any("erro", err))
+ }
+ } else {
+ // If the OssMultipartUploadResult exists in cache, use it to initiate a multipart upload
+ imur = imurWrapper.Imur
+ }
+
+ if err != nil {
+ return fmt.Errorf("failed to initiate multipart upload: %w", err)
+ }
+
+ err = w.streamUploadWithRetry(ctx, pointer, partSize, w.bucket, imur, w.config.Mirror.LfsConcurrency)
+ if err != nil {
+ return fmt.Errorf("failed to stream upload: %w", err)
+ }
+
+ return nil
+}
+
+func (w *AliyunOssLfSSyncWorker) streamUploadWithRetry(ctx context.Context, pointer *types.Pointer, partSize int64, bucket *oss.Bucket, imur oss.InitiateMultipartUploadResult, concurrency int) error {
+ var parts []oss.UploadPart
+ repoPath := ctx.Value(rk).(string)
+ eg := new(errgroup.Group)
+ // Use a channel to limit the number of concurrent goroutines
+ concurrencyChan := make(chan struct{}, concurrency)
+ contentLength := pointer.Size
+ url := pointer.DownloadURL
+ partNumberO := 1
+ for i := 0; i < concurrency; i++ {
+ concurrencyChan <- struct{}{}
+ }
+ totalParts := int64(math.Ceil(float64(contentLength) / float64(partSize)))
+ for offsetO := int64(0); offsetO < contentLength; offsetO += partSize {
+ offset := offsetO
+ partNumber := partNumberO
+ // Use errgroup.Group to get the first error of the goroutines
+ <-concurrencyChan
+ eg.Go(func() error {
+ defer func() {
+ concurrencyChan <- struct{}{}
+ }()
+ end := offset + partSize - 1
+ if end > contentLength {
+ end = contentLength - 1
+ }
+ // Check if the part of lfs file has already been uploaded
+ // If the part of lfs file has already been uploaded, skip it
+ synced, err := w.syncCache.IsLfsPartSynced(ctx, repoPath, pointer.Oid, partNumber)
+ if err != nil {
+ slog.Error("failed to get LFS part synced from cache", slog.String("error", err.Error()), slog.Any("pointer", pointer), slog.Int64("part_number", int64(partNumber)), slog.Any("repo_path", repoPath))
+ }
+ if synced {
+ return nil
+ }
+ slog.Info("uploading part", slog.Any("oid", pointer.Oid), slog.Int("part_number", partNumber))
+ part, err := downloadAndUploadWithRetry(ctx, url, offset, end, bucket, imur, partNumber)
+ if err != nil {
+ slog.Error("failed to uploading part", slog.Int("part_number", partNumber), slog.Any("error", err))
+ return fmt.Errorf("failed uploading part %d: %w", partNumber, err)
+ }
+ w.mu.Lock()
+ parts = append(parts, part)
+ w.mu.Unlock()
+ // Add the partNumber to cache to indicate that it has been uploaded
+ err = w.syncCache.CacheLfsSyncAddPart(ctx, repoPath, pointer.Oid, partNumber)
+ if err != nil {
+ slog.Error("failed caching part", slog.Any("oid", pointer.Oid), slog.Int("part_number", partNumber), slog.Any("error", err))
+ return fmt.Errorf("failed caching part: %w", err)
+ }
+ // Count the progress of the lfs file and save it in cache
+ uploadedPartCount, err := w.syncCache.LfsPartSyncedCount(ctx, repoPath, pointer.Oid)
+ if err != nil {
+ slog.Error("failed to get uploaded part count", slog.Any("oid", pointer.Oid), slog.Any("error", err))
+ return fmt.Errorf("failed to get uploaded part count: %w", err)
+ }
+ progress := float64(uploadedPartCount) / float64(totalParts) * 100
+ err = w.syncCache.CacheLfsSyncFileProgress(ctx, repoPath, pointer.Oid, int(progress))
+ if err != nil {
+ slog.Error("failed to cache lfs upload progress", slog.Any("oid", pointer.Oid), slog.Any("error", err))
+ return fmt.Errorf("failed to cache lfs upload progress: %w", err)
+ }
+ return nil
+ })
+
+ partNumberO++
+ }
+ err := eg.Wait()
+ if err != nil {
+ slog.Error("failed to download and upload lfs file", slog.Any("error", err))
+ return fmt.Errorf("error uploading parts: %w", err)
+ }
+
+ _, err = bucket.CompleteMultipartUpload(imur, parts)
+ if err != nil {
+ slog.Error("error completing multipart upload", slog.Any("error", err))
+ return fmt.Errorf("error completing multipart upload: %w", err)
+ }
+ slog.Info("completed multipart upload")
+
+ return nil
+}
+
+func downloadAndUploadWithRetry(ctx context.Context, url string, start, end int64, bucket *oss.Bucket, imur oss.InitiateMultipartUploadResult, partNumber int) (oss.UploadPart, error) {
+ var part oss.UploadPart
+ workerId := ctx.Value(wk).(int)
+ repoPath := ctx.Value(rk).(string)
+ for attempt := 1; attempt <= maxRetries; attempt++ {
+ slog.Info("Downloading range", slog.Any("worker_id", workerId), slog.Any("repo_path", repoPath), slog.Any("attempt", attempt), slog.Any("url", url), slog.Any("start", start), slog.Any("end", end))
+ resp, err := downloadRange(url, start, end)
+ if err != nil {
+ slog.Error("Error downloading range", slog.Any("worker_id", workerId), slog.Any("repo_path", repoPath), slog.Any("attempt", attempt), slog.Any("start", start), slog.Any("end", end), slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+ defer resp.Body.Close()
+
+ slog.Info("Uploading part", slog.Any("worker_id", workerId), slog.Any("repo_path", repoPath), slog.Any("attempt", attempt), slog.Any("url", url), slog.Any("part_number", partNumber))
+ part, err = bucket.UploadPart(imur, resp.Body, end-start+1, partNumber)
+ if err != nil {
+ slog.Error("Error uploading part", slog.Any("worker_id", workerId), slog.Any("repo_path", repoPath), slog.Any("attempt", attempt), slog.Any("part_number", partNumber), slog.Any("error", err))
+ time.Sleep(2 * time.Second)
+ continue
+ }
+
+ return part, nil
+ }
+
+ return part, fmt.Errorf("failed to download or upload part %d after %d attempts", partNumber, maxRetries)
+}
+
+func downloadRange(u string, start, end int64) (*http.Response, error) {
+ req, err := http.NewRequest("GET", u, nil)
+ if err != nil {
+ return nil, err
+ }
+ parsedURL, err := url.Parse(u)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse LFS download URL: %v", err)
+ }
+
+ req.Header.Set("Host", parsedURL.Host)
+ req.Header.Set("Accept", "application/vnd.git-lfs+json")
+ req.Header.Set("Content-Type", "application/vnd.git-lfs+json; charset=utf-8")
+ req.Header.Set("User-Agent", "git-lfs/3.5.1")
+
+ rangeHeader := fmt.Sprintf("bytes=%d-%d", start, end)
+ req.Header.Set("Range", rangeHeader)
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status code %d", resp.StatusCode)
+ }
+
+ return resp, nil
+}
+
+
+
package lfssyncer
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "net/url"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/minio/minio-go/v7"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/builder/store/s3"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+type MinioLFSSyncWorker struct {
+ mq queue.PriorityQueue
+ tasks chan queue.MirrorTask
+ wg sync.WaitGroup
+ mirrorStore database.MirrorStore
+ lfsMetaObjectStore database.LfsMetaObjectStore
+ s3Client s3.Client
+ config *config.Config
+ repoStore database.RepoStore
+ numWorkers int
+}
+
+func NewMinioLFSSyncWorker(config *config.Config, numWorkers int) (*MinioLFSSyncWorker, error) {
+ var err error
+ w := &MinioLFSSyncWorker{}
+ w.numWorkers = numWorkers
+ w.s3Client, err = s3.NewMinio(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to init s3 client for code,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ w.mirrorStore = database.NewMirrorStore()
+ w.repoStore = database.NewRepoStore()
+ w.lfsMetaObjectStore = database.NewLfsMetaObjectStore()
+ w.config = config
+ mq, err := queue.GetPriorityQueueInstance()
+ if err != nil {
+ return nil, fmt.Errorf("fail to get priority queue: %w", err)
+ }
+ w.mq = mq
+ w.tasks = make(chan queue.MirrorTask)
+ return w, nil
+}
+
+func (w *MinioLFSSyncWorker) Run() {
+ for i := 1; i <= w.numWorkers; i++ {
+ w.wg.Add(1)
+ go w.worker(i)
+ }
+ go w.dispatcher()
+ w.wg.Wait()
+}
+
+func (w *MinioLFSSyncWorker) dispatcher() {
+ for {
+ task := w.mq.PopLfsMirror()
+ if task != nil {
+ w.tasks <- *task
+ }
+ }
+}
+
+func (w *MinioLFSSyncWorker) worker(id int) {
+ defer w.wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ w.wg.Add(1)
+ go w.worker(id)
+ slog.Info("worker ecovered from panic ", slog.Int("workerId", id))
+ }
+ }()
+ slog.Info("worker start", slog.Int("workerId", id))
+ for {
+ task := <-w.tasks
+ ctx := context.Background()
+ mirror, err := w.mirrorStore.FindByID(ctx, task.MirrorID)
+ if err != nil {
+ slog.Error("fail to get mirror", slog.Int("workerId", id), slog.String("error", err.Error()))
+ continue
+ }
+ repo, err := w.repoStore.FindById(ctx, mirror.RepositoryID)
+ if err != nil {
+ slog.Error("fail to get repository", slog.Int("workerId", id), slog.String("error", err.Error()))
+ continue
+ }
+ err = w.SyncLfs(ctx, id, mirror)
+ if err != nil {
+ repo.SyncStatus = types.SyncStatusFailed
+ mirror.LastMessage = err.Error()
+ err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, repo)
+ if err != nil {
+ slog.Error("fail to update mirror and repository", slog.Int("workerId", id), slog.Any("error", err))
+ }
+ slog.Error("fail to sync lfs", slog.Int("workerId", id), slog.String("error", err.Error()))
+ continue
+ }
+
+ repo.SyncStatus = types.SyncStatusCompleted
+ mirror.LastUpdatedAt = time.Now()
+ err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, repo)
+ if err != nil {
+ slog.Error("fail to update mirror and repository", slog.Int("workerId", id), slog.Any("error", err))
+ }
+ slog.Info("sync lfs completed", slog.Int("workerId", id))
+ }
+}
+
+func (w *MinioLFSSyncWorker) SyncLfs(ctx context.Context, workerId int, mirror *database.Mirror) error {
+ var pointers []*types.Pointer
+ lfsMetaObjects, err := w.lfsMetaObjectStore.FindByRepoID(ctx, mirror.Repository.ID)
+ if err != nil {
+ slog.Error("fail to get lfs meta objects", slog.Int("workerId", workerId), slog.String("error", err.Error()))
+ return fmt.Errorf("fail to get lfs meta objects: %w", err)
+ }
+ for _, lfsMetaObject := range lfsMetaObjects {
+ pointers = append(pointers, &types.Pointer{
+ Oid: lfsMetaObject.Oid,
+ Size: lfsMetaObject.Size,
+ })
+ }
+
+ pointers, err = w.GetLFSDownloadURLs(ctx, mirror, pointers)
+ if err != nil {
+ return fmt.Errorf("fail to get LFS download URL: %w", err)
+ }
+ err = w.DownloadAndUploadLFSFiles(ctx, mirror, pointers)
+ if err != nil {
+ return fmt.Errorf("fail to download and upload LFS files: %w", err)
+ }
+ return nil
+}
+
+func (w *MinioLFSSyncWorker) GetLFSDownloadURLs(ctx context.Context, mirror *database.Mirror, pointers []*types.Pointer) ([]*types.Pointer, error) {
+ var (
+ resPointers []*types.Pointer
+ lfsAPIURL string
+ )
+ requestPayload := types.LFSBatchRequest{
+ Operation: "download",
+ }
+
+ for _, pointer := range pointers {
+ requestPayload.Objects = append(requestPayload.Objects, types.LFSBatchObject{
+ Oid: pointer.Oid,
+ Size: pointer.Size,
+ })
+ }
+ requestPayload.HashAlog = "sha256"
+ requestPayload.Transfers = []string{"lfs-standalone-file", "basic", "bash"}
+
+ if strings.HasSuffix(mirror.SourceUrl, ".git") {
+ lfsAPIURL = mirror.SourceUrl + "/info/lfs/objects/batch"
+ } else {
+ lfsAPIURL = mirror.SourceUrl + ".git/info/lfs/objects/batch"
+ }
+
+ payload, err := json.Marshal(requestPayload)
+ if err != nil {
+ return resPointers, fmt.Errorf("failed to marshal request payload: %v", err)
+ }
+
+ req, err := http.NewRequest("POST", lfsAPIURL, bytes.NewReader(payload))
+ if err != nil {
+ return resPointers, fmt.Errorf("failed to create LFS batch request: %v", err)
+ }
+
+ parsedURL, err := url.Parse(lfsAPIURL)
+ if err != nil {
+ return resPointers, fmt.Errorf("failed to parse LFS API URL: %v", err)
+ }
+
+ req.Header.Set("Host", parsedURL.Host)
+ req.Header.Set("Accept", "application/vnd.git-lfs+json")
+ req.Header.Set("Content-Type", "application/vnd.git-lfs+json; charset=utf-8")
+ req.Header.Set("User-Agent", "git-lfs/3.5.1")
+
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ return resPointers, fmt.Errorf("failed to send LFS batch request: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return resPointers, fmt.Errorf("failed to get LFS download URL, status code: %d", resp.StatusCode)
+ }
+
+ var batchResponse types.LFSBatchResponse
+ err = json.NewDecoder(resp.Body).Decode(&batchResponse)
+ if err != nil {
+ return resPointers, fmt.Errorf("failed to decode LFS batch response: %v", err)
+ }
+
+ if len(batchResponse.Objects) == 0 {
+ return resPointers, fmt.Errorf("no objects found in LFS batch response")
+ }
+ for _, object := range batchResponse.Objects {
+ resPointers = append(resPointers, &types.Pointer{
+ Oid: object.Oid,
+ Size: object.Size,
+ DownloadURL: object.Actions.Download.Href,
+ })
+ }
+
+ return resPointers, nil
+}
+
+func (w *MinioLFSSyncWorker) DownloadAndUploadLFSFiles(ctx context.Context, mirror *database.Mirror, pointers []*types.Pointer) error {
+ var finishedLFSFileCount int
+ lfsFilesCount := len(pointers)
+ for _, pointer := range pointers {
+ objectKey := filepath.Join("lfs", pointer.RelativePath())
+ fileInfo, err := w.s3Client.StatObject(ctx, w.config.S3.Bucket, objectKey, minio.StatObjectOptions{})
+ if err != nil && err.Error() != "The specified key does not exist." {
+ slog.Error("failed to check if LFS file exists", slog.Any("error", err))
+ continue
+ }
+ if (err != nil && err.Error() != "The specified key does not exist.") || fileInfo.Size != pointer.Size {
+ err = w.DownloadAndUploadLFSFile(ctx, mirror, pointer)
+ if err != nil {
+ slog.Error("failed to download and upload LFS file", slog.Any("error", err))
+ }
+ }
+
+ lfsMetaObject := database.LfsMetaObject{
+ Size: pointer.Size,
+ Oid: pointer.Oid,
+ RepositoryID: mirror.Repository.ID,
+ Existing: true,
+ }
+ _, err = w.lfsMetaObjectStore.UpdateOrCreate(ctx, lfsMetaObject)
+ if err != nil {
+ slog.Error("failed to update or create LFS meta object", slog.Any("error", err))
+ return fmt.Errorf("failed to update or create LFS meta object: %w", err)
+ }
+ slog.Info("finish to download and upload LFS file", slog.Any("objectKey", objectKey))
+ finishedLFSFileCount += 1
+ mirror.Progress = int8(finishedLFSFileCount * 100 / lfsFilesCount)
+ err = w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror progress: %w", err)
+ }
+ }
+ mirror.Status = types.MirrorFinished
+ err := w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror status: %w", err)
+ }
+ return nil
+}
+
+func (w *MinioLFSSyncWorker) DownloadAndUploadLFSFile(ctx context.Context, mirror *database.Mirror, pointer *types.Pointer) error {
+ objectKey := filepath.Join("lfs", pointer.RelativePath())
+ slog.Info("downloading LFS file from", slog.Any("url", pointer.DownloadURL))
+
+ req, err := http.NewRequest("GET", pointer.DownloadURL, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create download request: %w", err)
+ }
+
+ parsedURL, err := url.Parse(pointer.DownloadURL)
+ if err != nil {
+ return fmt.Errorf("failed to parse LFS API URL: %v", err)
+ }
+
+ req.Header.Set("Host", parsedURL.Host)
+ req.Header.Set("Accept", "application/vnd.git-lfs+json")
+ req.Header.Set("Content-Type", "application/vnd.git-lfs+json; charset=utf-8")
+ req.Header.Set("User-Agent", "git-lfs/3.5.1")
+
+ client := &http.Client{}
+ resp, err := client.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to download LFS file: %w", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("failed to download LFS file: %s", resp.Status)
+ }
+ slog.Info("uploading LFS file", slog.Any("object_key", objectKey))
+ uploadInfo, err := w.s3Client.PutObject(ctx, w.config.S3.Bucket, objectKey, resp.Body, resp.ContentLength, minio.PutObjectOptions{})
+ if err != nil {
+ return fmt.Errorf("failed to upload to Minio: %w", err)
+ }
+
+ if uploadInfo.Size != pointer.Size {
+ return fmt.Errorf("uploaded file size does not match expected size: %d != %d", uploadInfo.Size, pointer.Size)
+ }
+
+ return nil
+}
+
+
+
package mirror
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+type MirrorPriorityQueue struct {
+ mq queue.PriorityQueue
+ tasks chan queue.MirrorTask
+ numWorkers int
+}
+
+func NewMirrorPriorityQueue(config *config.Config) (*MirrorPriorityQueue, error) {
+ s := &MirrorPriorityQueue{}
+ mq, err := queue.GetPriorityQueueInstance()
+ if err != nil {
+ return nil, fmt.Errorf("fail to get priority queue: %w", err)
+ }
+ s.mq = mq
+ s.tasks = make(chan queue.MirrorTask)
+ s.numWorkers = config.Mirror.WorkerNumber
+ return s, nil
+}
+
+func (ms *MirrorPriorityQueue) EnqueueMirrorTasks() {
+ mirrorStore := database.NewMirrorStore()
+ mirrors, err := mirrorStore.ToSyncRepo(context.Background())
+ if err != nil {
+ slog.Error("fail to get mirror to sync", slog.String("error", err.Error()))
+ return
+ }
+
+ for _, mirror := range mirrors {
+ ms.mq.PushRepoMirror(&queue.MirrorTask{
+ MirrorID: mirror.ID,
+ Priority: queue.Priority(mirror.Priority),
+ CreatedAt: mirror.CreatedAt.Unix(),
+ })
+ mirror.Status = types.MirrorWaiting
+ err = mirrorStore.Update(context.Background(), &mirror)
+ if err != nil {
+ slog.Error("fail to update mirror status", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ continue
+ }
+ }
+
+ mirrors, err = mirrorStore.ToSyncLfs(context.Background())
+ if err != nil {
+ slog.Error("fail to get mirror to sync", slog.String("error", err.Error()))
+ return
+ }
+
+ for _, mirror := range mirrors {
+ ms.mq.PushLfsMirror(&queue.MirrorTask{
+ MirrorID: mirror.ID,
+ Priority: queue.Priority(mirror.Priority),
+ CreatedAt: mirror.CreatedAt.Unix(),
+ })
+ mirror.Status = types.MirrorWaiting
+ err = mirrorStore.Update(context.Background(), &mirror)
+ if err != nil {
+ slog.Error("fail to update mirror status", slog.Int64("mirrorId", mirror.ID), slog.String("error", err.Error()))
+ continue
+ }
+ }
+}
+
+
+
package queue
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/redis/go-redis/v9"
+ "opencsg.com/csghub-server/builder/store/cache"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type Priority int
+
+func (p Priority) Int() int { return int(p) }
+
+const (
+ HighPriority Priority = 3
+ MediumPriority Priority = 2
+ LowPriority Priority = 1
+)
+
+var PriorityMap = map[types.MirrorPriority]Priority{
+ types.HighMirrorPriority: HighPriority,
+ types.MediumMirrorPriority: MediumPriority,
+ types.LowMirrorPriority: LowPriority,
+}
+
+const (
+ repoQueueName = "repo_mirror_queue"
+ lfsQueueName = "lfs_mirror_queue"
+)
+
+type MirrorTask struct {
+ MirrorID int64 `json:"mirror_id"`
+ Priority Priority `json:"priority"`
+ CreatedAt int64 `json:"created_at"`
+ MirrorToken string `json:"mirror_token"`
+}
+
+type MirrorQueue struct {
+ redis *cache.Cache
+ QueueName string
+}
+
+func (m *MirrorTask) MarshalBinary() ([]byte, error) {
+ return json.Marshal(m)
+}
+
+func (m *MirrorTask) UnmarshalBinary(data []byte) error {
+ return json.Unmarshal(data, m)
+}
+
+func (mq *MirrorQueue) Push(t *MirrorTask) {
+ if t.CreatedAt == 0 {
+ t.CreatedAt = time.Now().Unix()
+ }
+ _ = mq.redis.ZAdd(context.Background(), mq.QueueName, redis.Z{
+ Score: float64(t.CreatedAt) * float64(t.Priority),
+ Member: t,
+ })
+}
+
+func (mq *MirrorQueue) Pop() *MirrorTask {
+ r, err := mq.redis.BZPopMax(context.Background(), mq.QueueName)
+ if err != nil {
+ return nil
+ }
+ var task MirrorTask
+ _ = json.Unmarshal([]byte(r.Member.(string)), &task)
+ return &task
+}
+
+type PriorityQueue interface {
+ PushRepoMirror(mt *MirrorTask)
+ PopRepoMirror() *MirrorTask
+ PushLfsMirror(mt *MirrorTask)
+ PopLfsMirror() *MirrorTask
+}
+
+type priorityQueueImpl struct {
+ RepoMirrorQueue MirrorQueue
+ LfsMirrorQueue MirrorQueue
+}
+
+var (
+ instance PriorityQueue
+ once sync.Once
+ err error
+ c *config.Config
+)
+
+func NewPriorityQueue(ctx context.Context, config *config.Config) (PriorityQueue, error) {
+ redis, err := cache.NewCache(ctx, cache.RedisConfig{
+ Addr: config.Redis.Endpoint,
+ Username: config.Redis.User,
+ Password: config.Redis.Password,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("initializing redis: %w", err)
+ }
+ mq := &priorityQueueImpl{
+ RepoMirrorQueue: MirrorQueue{
+ redis: redis,
+ QueueName: repoQueueName,
+ },
+ LfsMirrorQueue: MirrorQueue{
+ redis: redis,
+ QueueName: lfsQueueName,
+ },
+ }
+ return mq, nil
+}
+
+func (pq *priorityQueueImpl) PushRepoMirror(mt *MirrorTask) {
+ pq.RepoMirrorQueue.Push(mt)
+}
+
+func (pq *priorityQueueImpl) PopRepoMirror() *MirrorTask {
+ return pq.RepoMirrorQueue.Pop()
+}
+
+func (pq *priorityQueueImpl) PushLfsMirror(mt *MirrorTask) {
+ pq.LfsMirrorQueue.Push(mt)
+}
+
+func (pq *priorityQueueImpl) PopLfsMirror() *MirrorTask {
+ return pq.LfsMirrorQueue.Pop()
+}
+
+func GetPriorityQueueInstance() (PriorityQueue, error) {
+ once.Do(func() {
+ c, err = config.LoadConfig()
+ instance, err = NewPriorityQueue(context.Background(), c)
+ })
+ if err != nil {
+ return nil, err
+ }
+ return instance, nil
+}
+
+
+
package mirror
+
+import (
+ "context"
+
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/mirror/queue"
+ "opencsg.com/csghub-server/mirror/reposyncer"
+)
+
+type RepoSyncWorker interface {
+ Run()
+ SyncRepo(ctx context.Context, task queue.MirrorTask) error
+}
+
+func NewRepoSyncWorker(config *config.Config, numWorkers int) (RepoSyncWorker, error) {
+ return reposyncer.NewLocalMirrorWoker(config, numWorkers)
+}
+
+
+
package reposyncer
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "go.temporal.io/sdk/client"
+ "golang.org/x/time/rate"
+ "opencsg.com/csghub-server/api/workflow"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/gitserver/gitaly"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/mirror/queue"
+)
+
+type LocalMirrorWoker struct {
+ mq queue.PriorityQueue
+ tasks chan queue.MirrorTask
+ numWorkers int
+ wg sync.WaitGroup
+ saas bool
+ mirrorStore database.MirrorStore
+ lfsMetaObjectStore database.LfsMetaObjectStore
+ repoStore database.RepoStore
+ git gitserver.GitServer
+ config *config.Config
+ ratelimiter *rate.Limiter
+}
+
+func NewLocalMirrorWoker(config *config.Config, numWorkers int) (*LocalMirrorWoker, error) {
+ var err error
+ w := &LocalMirrorWoker{}
+ w.numWorkers = numWorkers
+ w.git, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ w.mirrorStore = database.NewMirrorStore()
+ w.repoStore = database.NewRepoStore()
+ w.lfsMetaObjectStore = database.NewLfsMetaObjectStore()
+ w.saas = config.Saas
+ w.config = config
+ mq, err := queue.GetPriorityQueueInstance()
+ if err != nil {
+ return nil, fmt.Errorf("fail to get priority queue: %w", err)
+ }
+ w.mq = mq
+ w.tasks = make(chan queue.MirrorTask)
+ w.numWorkers = numWorkers
+ w.ratelimiter = rate.NewLimiter(rate.Limit(config.Mirror.RateLimit), config.Mirror.RateBucketCapacity)
+ return w, nil
+}
+
+func (w *LocalMirrorWoker) Run() {
+ for i := 1; i <= w.numWorkers; i++ {
+ w.wg.Add(1)
+ go w.worker(i)
+ }
+ go w.dispatcher()
+ w.wg.Wait()
+}
+
+func (w *LocalMirrorWoker) dispatcher() {
+ for {
+ task := w.mq.PopRepoMirror()
+ if task != nil {
+ w.tasks <- *task
+ }
+ }
+}
+
+func (w *LocalMirrorWoker) worker(id int) {
+ defer w.wg.Done()
+ defer func() {
+ if r := recover(); r != nil {
+ w.wg.Add(1)
+ go w.worker(id)
+ slog.Info("worker ecovered from panic ", slog.Int("workerId", id), slog.Any("r", r))
+ }
+ }()
+ slog.Info("worker start", slog.Int("workerId", id))
+ for {
+ err := w.ratelimiter.Wait(context.Background())
+ if err != nil {
+ slog.Error("Error waiting for rate limiter:", slog.Any("error", err))
+ continue
+ }
+ task := <-w.tasks
+ slog.Info("start to mirror", slog.Int64("mirrorId", task.MirrorID), slog.Int("priority", task.Priority.Int()), slog.Int("workerId", id))
+ err = w.SyncRepo(context.Background(), task)
+ if err != nil {
+ slog.Info("fail to mirror", slog.Int64("mirrorId", task.MirrorID), slog.Int("priority", task.Priority.Int()), slog.Int("workerId", id), slog.String("error", err.Error()))
+ }
+ slog.Info("finish to mirror", slog.Int64("mirrorId", task.MirrorID), slog.Int("priority", task.Priority.Int()), slog.Int("workerId", id))
+ }
+}
+
+func (w *LocalMirrorWoker) SyncRepo(ctx context.Context, task queue.MirrorTask) error {
+ mirror, err := w.mirrorStore.FindByID(ctx, task.MirrorID)
+ if err != nil {
+ return fmt.Errorf("failed to get mirror: %v", err)
+ }
+ mirror.Status = types.MirrorRunning
+ mirror.Priority = types.LowMirrorPriority
+ err = w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror status: %v", err)
+ }
+ if mirror.Repository == nil {
+ return fmt.Errorf("mirror repository is nil")
+ }
+ namespace := strings.Split(mirror.Repository.Path, "/")[0]
+ name := strings.Split(mirror.Repository.Path, "/")[1]
+
+ slog.Info("Start to sync mirror repo", "repo_type", mirror.Repository.RepositoryType, "namespace", namespace, "name", name)
+ req := gitserver.MirrorSyncReq{
+ Namespace: namespace,
+ Name: name,
+ CloneUrl: mirror.SourceUrl,
+ Username: mirror.Username,
+ AccessToken: mirror.AccessToken,
+ RepoType: mirror.Repository.RepositoryType,
+ }
+ if task.MirrorToken != "" {
+ req.MirrorToken = task.MirrorToken
+ }
+ err = w.git.MirrorSync(ctx, req)
+
+ if err != nil {
+ mirror.Status = types.MirrorFailed
+ mirror.LastMessage = fmt.Sprintf("failed mirror remote repo in git server:%s", err.Error())
+ updateErr := w.mirrorStore.Update(ctx, mirror)
+ if updateErr != nil {
+ return fmt.Errorf("failed to update mirror: %w", updateErr)
+ }
+ return fmt.Errorf("failed mirror remote repo in git server: %v", err)
+ }
+ slog.Info("Mirror remote repo in git server successfully", "repo_type", mirror.Repository.RepositoryType, "namespace", namespace, "name", name)
+
+ resp, err := w.git.GetRepo(ctx, gitserver.GetRepoReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: mirror.Repository.RepositoryType,
+ })
+ if err != nil {
+ mirror.Status = types.MirrorFailed
+ mirror.LastMessage = fmt.Sprintf("failed to get repo detail:%s", err.Error())
+ err = w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror status: %w", err)
+ }
+ return fmt.Errorf("failed to get repo default branch: %w", err)
+ }
+ parts := strings.Split(string(resp.DefaultBranch), "/")
+ branch := parts[len(parts)-1]
+
+ mirror.Repository.DefaultBranch = branch
+ mirror.Repository.SyncStatus = types.SyncStatusInProgress
+ _, err = w.repoStore.UpdateRepo(ctx, *mirror.Repository)
+ if err != nil {
+ return fmt.Errorf("failed to update repo sync status to in progress: %w", err)
+ }
+ slog.Info("Update repo default branch successfully", slog.Any("repo_type", mirror.Repository.RepositoryType), slog.Any("namespace", namespace), slog.Any("name", name))
+ slog.Info("Start to sync lfs files", "repo_type", mirror.Repository.RepositoryType, "namespace", namespace, "name", name)
+ lfsFileCount, err := w.generateLfsMetaObjects(ctx, mirror)
+ if err != nil {
+ mirror.Status = types.MirrorIncomplete
+ mirror.LastMessage = err.Error()
+ mirror.Repository.SyncStatus = types.SyncStatusFailed
+ err = w.mirrorStore.UpdateMirrorAndRepository(ctx, mirror, mirror.Repository)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror and repository: %w", err)
+ }
+ return fmt.Errorf("failed to generate lfs meta objects: %v", err)
+ }
+ if lfsFileCount > 0 {
+ mirror.Status = types.MirrorRepoSynced
+ w.mq.PushLfsMirror(&queue.MirrorTask{
+ MirrorID: mirror.ID,
+ Priority: queue.Priority(mirror.Priority),
+ CreatedAt: mirror.CreatedAt.Unix(),
+ MirrorToken: task.MirrorToken,
+ })
+ } else {
+ mirror.Status = types.MirrorFinished
+ mirror.Repository.SyncStatus = types.SyncStatusCompleted
+ _, err = w.repoStore.UpdateRepo(ctx, *mirror.Repository)
+ if err != nil {
+ return fmt.Errorf("failed to update repo sync status to completed: %w", err)
+ }
+ }
+ // Update mirror last updated at
+ mirror.LastUpdatedAt = time.Now()
+
+ err = w.mirrorStore.Update(ctx, mirror)
+ if err != nil {
+ return fmt.Errorf("failed to update mirror: %w", err)
+ }
+
+ // Trigger git callback
+
+ // Get repo last commit
+ commit, err := w.git.GetRepoLastCommit(ctx, gitserver.GetRepoLastCommitReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: mirror.Repository.RepositoryType,
+ Ref: branch,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to get repo last commit: %w", err)
+ }
+
+ callback, err := w.git.GetDiffBetweenTwoCommits(ctx, gitserver.GetDiffBetweenTwoCommitsReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: mirror.Repository.RepositoryType,
+ Ref: branch,
+ LeftCommitId: gitaly.SHA1EmptyTreeID,
+ RightCommitId: commit.ID,
+ Private: mirror.Repository.Private,
+ })
+ if err != nil {
+ return fmt.Errorf("failed to get diff between two commits: %w", err)
+ }
+ callback.Ref = branch
+
+ //start workflow to handle push request
+ workflowClient := workflow.GetWorkflowClient()
+ workflowOptions := client.StartWorkflowOptions{
+ TaskQueue: workflow.HandlePushQueueName,
+ }
+
+ we, err := workflowClient.ExecuteWorkflow(ctx, workflowOptions, workflow.HandlePushWorkflow,
+ callback,
+ w.config,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to handle git push callback: %w", err)
+ }
+
+ slog.Info("start handle push workflow", slog.String("workflow_id", we.GetID()), slog.Any("req", callback))
+
+ return nil
+}
+
+func (c *LocalMirrorWoker) generateLfsMetaObjects(ctx context.Context, mirror *database.Mirror) (int, error) {
+ var lfsMetaObjects []database.LfsMetaObject
+ namespace := strings.Split(mirror.Repository.Path, "/")[0]
+ name := strings.Split(mirror.Repository.Path, "/")[1]
+ branches, err := c.git.GetRepoBranches(ctx, gitserver.GetBranchesReq{
+ Namespace: namespace,
+ Name: name,
+ RepoType: mirror.Repository.RepositoryType,
+ })
+ if err != nil {
+ return 0, fmt.Errorf("failed to get repo branches: %v", err)
+ }
+ for _, branch := range branches {
+ lfsPointers, err := c.getAllLfsPointersByRef(ctx, mirror.Repository.RepositoryType, namespace, name, branch.Name)
+ if err != nil {
+ return 0, fmt.Errorf("failed to get all lfs pointers: %v", err)
+ }
+ for _, lfsPointer := range lfsPointers {
+ lfsMetaObjects = append(lfsMetaObjects, database.LfsMetaObject{
+ Size: lfsPointer.FileSize,
+ Oid: lfsPointer.FileOid,
+ RepositoryID: mirror.Repository.ID,
+ Existing: true,
+ })
+ }
+ }
+ lfsMetaObjects = removeDuplicateLfsMetaObject(lfsMetaObjects)
+
+ if len(lfsMetaObjects) > 0 {
+ err = c.lfsMetaObjectStore.BulkUpdateOrCreate(ctx, lfsMetaObjects)
+ if err != nil {
+ return 0, fmt.Errorf("failed to bulk update or create lfs meta objects: %v", err)
+ }
+ }
+
+ return len(lfsMetaObjects), nil
+}
+
+func (c *LocalMirrorWoker) getAllLfsPointersByRef(ctx context.Context, RepoType types.RepositoryType, namespace, name, ref string) ([]*types.LFSPointer, error) {
+ return c.git.GetRepoAllLfsPointers(ctx, gitserver.GetRepoAllFilesReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: ref,
+ RepoType: RepoType,
+ })
+}
+
+func removeDuplicateLfsMetaObject(objects []database.LfsMetaObject) []database.LfsMetaObject {
+ seen := make(map[string]bool)
+ uniqueObjects := []database.LfsMetaObject{}
+
+ for _, obj := range objects {
+ key := obj.Oid + "_" + strconv.Itoa(int(obj.RepositoryID))
+ if !seen[key] {
+ uniqueObjects = append(uniqueObjects, obj)
+ seen[key] = true
+ }
+ }
+
+ return uniqueObjects
+}
+
+
+
package checker
+
+import (
+ "io"
+ "path"
+ "slices"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var knownImageFileExts = []string{".png", ".jpg", ".jpeg", ".gif", ".tif", ".tiff", ".svg", ".bmp", ".webp"}
+var knownTextFileExts = []string{".md", ".txt", ".csv", ".json", ".jsonl", ".html",
+ //code file types
+ ".cs", ".js", ".ts", ".py", ".php", ".java", ".c", ".cpp", ".go", ".rb", ".sh"}
+
+type FileChecker interface {
+ Run(reader io.Reader) (types.SensitiveCheckStatus, string)
+}
+
+// GetFileChecker returns a FileChecker for a given file based on its type and path.
+//
+// The checkers are chosen as follows:
+// - folder: FolderChecker
+// - LFS files: LfsFileChecker
+// - unknown files: UnkownFileChecker
+// - image files (with extensions .png, .jpg, .jpeg, .gif, .tif, .tiff, .svg, .bmp, .webp): ImageFileChecker
+// - text files (with extensions .md, .txt, .csv, .json, .jsonl, .html, .cs, .js, .ts, .py, .php, .java, .c, .cpp, .go, .rb, .sh): TextFileChecker
+func GetFileChecker(fileType string, filePath, lfsRelativePath string) FileChecker {
+
+ if fileType == "folder" {
+ return &FolderChecker{}
+ }
+
+ if lfsRelativePath != "" {
+ return &LfsFileChecker{}
+ }
+
+ ext := path.Ext(filePath)
+ if len(ext) == 0 {
+ return &UnkownFileChecker{}
+ }
+
+ if slices.ContainsFunc(knownImageFileExts, func(imageExt string) bool {
+ return strings.EqualFold(ext, imageExt)
+ }) {
+ return NewImageFileChecker()
+ }
+
+ if slices.ContainsFunc(knownTextFileExts, func(textExt string) bool {
+ return strings.EqualFold(ext, textExt)
+ }) {
+ return NewTextFileChecker()
+ }
+
+ return &UnkownFileChecker{}
+}
+
+type ImageFileChecker struct {
+ checker sensitive.SensitiveChecker
+}
+
+func NewImageFileChecker() FileChecker {
+ return &ImageFileChecker{
+ checker: contentChecker,
+ }
+}
+func (c *ImageFileChecker) Run(io.Reader) (types.SensitiveCheckStatus, string) {
+ //TODO:check image in the future
+ return types.SensitiveCheckSkip, "skip image file"
+}
+
+type LfsFileChecker struct {
+}
+
+func (c *LfsFileChecker) Run(io.Reader) (types.SensitiveCheckStatus, string) {
+ // dont need to check lfs file content
+ return types.SensitiveCheckSkip, "skip lfs file"
+}
+
+type FolderChecker struct {
+}
+
+func (c *FolderChecker) Run(reader io.Reader) (types.SensitiveCheckStatus, string) {
+ return types.SensitiveCheckSkip, "skip folder"
+}
+
+
+
package checker
+
+import (
+ "bufio"
+ "encoding/base64"
+ "strings"
+
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/common/config"
+)
+
+var contentChecker sensitive.SensitiveChecker
+var localWordChecker *DFA
+
+func Init(config *config.Config) {
+ if !config.SensitiveCheck.Enable {
+ panic("SensitiveCheck is not enable")
+ }
+ //init aliyun green checker
+ contentChecker = sensitive.NewAliyunGreenCheckerFromConfig(config)
+ //init local word checker
+ localWordChecker = NewDFA()
+
+ localWordChecker.BuildDFA(getSensitiveWordList(config.Moderation.EncodedSensitiveWords))
+}
+
+// InitWithContentChecker supports custom sensitive checker, this func mostly used in unit test
+func InitWithContentChecker(config *config.Config, checker sensitive.SensitiveChecker) {
+ if !config.SensitiveCheck.Enable {
+ panic("SensitiveCheck is not enable")
+ }
+
+ if checker == nil {
+ panic("param checker can not be nil")
+ }
+ contentChecker = checker
+ //init local word checker
+ localWordChecker = NewDFA()
+
+ localWordChecker.BuildDFA(getSensitiveWordList(config.Moderation.EncodedSensitiveWords))
+}
+
+func getSensitiveWordList(encodedWords string) []string {
+ r := base64.NewDecoder(base64.StdEncoding, strings.NewReader(encodedWords))
+ s := bufio.NewScanner(r)
+ s.Split(commaSplit)
+
+ var words []string
+ for s.Scan() {
+ words = append(words, s.Text())
+ }
+
+ return words
+}
+
+// Custom split function that splits on commas
+func commaSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
+ if len(data) == 0 {
+ // When there's no data, return normally
+ return 0, nil, nil
+ }
+
+ // Find the first comma
+ if i := strings.IndexByte(string(data), ','); i >= 0 {
+ // We've found a comma, return the part before it
+ return i + 1, data[:i], nil
+ }
+
+ // If we've reached EOF and there's data left, return it
+ if atEOF {
+ return len(data), data, nil
+ }
+
+ // If we haven't found a comma and we're not at EOF,
+ // we need more data
+ return 0, nil, nil
+}
+
+
+
package checker
+
+// DFA State
+type State struct {
+ transitions map[rune]*State
+ isEnd bool
+}
+
+// Deterministic Finite Automaton (DFA)
+type DFA struct {
+ root *State
+}
+
+// NewDFA initializes a new DFA
+func NewDFA() *DFA {
+ return &DFA{
+ root: &State{transitions: make(map[rune]*State)},
+ }
+}
+
+// BuildDFA constructs the DFA from a list of sensitive words
+func (d *DFA) BuildDFA(words []string) {
+ for _, word := range words {
+ current := d.root
+ for _, char := range word {
+ if next, exists := current.transitions[char]; exists {
+ current = next
+ } else {
+ newState := &State{transitions: make(map[rune]*State)}
+ current.transitions[char] = newState
+ current = newState
+ }
+ }
+ current.isEnd = true // Mark the end of a sensitive word
+ }
+}
+
+// ContainsSensitiveWord checks if the input text contains any sensitive words
+func (d *DFA) ContainsSensitiveWord(text string) bool {
+ runes := []rune(text)
+ current := d.root
+ for i := 0; i < len(runes); i++ {
+ char := runes[i]
+ if isIgnoredCharacter(char) {
+ continue
+ }
+ if next, exists := current.transitions[char]; exists {
+ current = next
+ if current.isEnd {
+ return true
+ }
+ } else {
+ current = d.root // Reset to the root state
+ if i > 0 {
+ // Check if the current character is the start of a new sensitive word
+ if next, exists := current.transitions[char]; exists {
+ current = next
+ }
+ }
+ }
+ }
+ return false
+}
+
+// isIgnoredCharacter checks if a character is in the specified ignored set
+func isIgnoredCharacter(c rune) bool {
+ ignoredCharacters := []rune{' ', '\u3000', '\t', '&', '%', '$', '@', '*', '！', '!', '#', '^', '~', '_', '—', '｜', '\'', '"', ';', '.', '，', ',', '?', '<', '>', '《', '》', '：', ':'}
+ for _, ignored := range ignoredCharacters {
+ if c == ignored {
+ return true
+ }
+ }
+ return false
+}
+
+
+
package checker
+
+import (
+ "bytes"
+ "context"
+ "io"
+ "log/slog"
+ "time"
+
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type TextFileChecker struct {
+ sensitive.SensitiveChecker
+}
+
+func NewTextFileChecker() *TextFileChecker {
+ return &TextFileChecker{
+ contentChecker,
+ }
+}
+
+func (c *TextFileChecker) Run(reader io.Reader) (types.SensitiveCheckStatus, string) {
+ //at most 1MB
+ reader = io.LimitReader(reader, 1024*1024)
+ const blockSize = 10 * 9000
+ // const blockSize = 3000
+ var bufs []bytes.Buffer
+ for {
+ buf := bytes.Buffer{}
+ var err error
+ var avaliableSize int64
+ if avaliableSize, err = io.CopyN(&buf, reader, blockSize); err != nil && err != io.EOF {
+ return types.SensitiveCheckException, "failed to read file content"
+ }
+ if avaliableSize > 0 {
+ bufs = append(bufs, buf)
+ }
+ //no more data to read
+ if avaliableSize < blockSize {
+ break
+ }
+ }
+ for _, buf := range bufs {
+ var result *sensitive.CheckResult
+ var err error
+ slog.Debug("check text", slog.String("scenario", string(sensitive.ScenarioCommentDetection)), slog.String("text", buf.String()))
+ //do local check first
+ txt := buf.String()
+ contains := localWordChecker.ContainsSensitiveWord(txt)
+ if contains {
+ return types.SensitiveCheckFail, "contains sensitive word"
+ }
+ //call remote checker
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ result, err = c.PassTextCheck(ctx, sensitive.ScenarioCommentDetection, txt)
+ cancel()
+ if err != nil {
+ return types.SensitiveCheckException, "call sensitive checker api failed"
+ }
+
+ if result.IsSensitive {
+ return types.SensitiveCheckFail, result.Reason
+ }
+ }
+
+ return types.SensitiveCheckPass, ""
+}
+
+
+
package checker
+
+import (
+ "bytes"
+ "io"
+ "log/slog"
+ "net/http"
+ "strings"
+
+ "opencsg.com/csghub-server/common/types"
+)
+
+// UnkownFileChecker handles the unknown file types (no file extension)
+//
+// Internally, it will read the first 512 bytes and detect the content type
+// and use the corresponding checker
+type UnkownFileChecker struct {
+}
+
+func (c *UnkownFileChecker) Run(reader io.Reader) (types.SensitiveCheckStatus, string) {
+ // read the first 512 bytes and detect the content type
+ buffer := make([]byte, 512)
+ n, err := reader.Read(buffer)
+ if err != nil {
+ return types.SensitiveCheckException, "failed to read file contents"
+ }
+
+ // remove zero bytes before detecting content type,
+ // see: https://gist.github.com/rayrutjes/db9b9ea8e02255d62ce2?permalink_comment_id=3418419#gistcomment-3418419
+ buffer = buffer[:n]
+ // Detect the file content type like text/plain, image/jpeg, etc
+ detectedType := http.DetectContentType(buffer)
+ switch {
+ case strings.HasPrefix(detectedType, "text"):
+ slog.Debug("use text file checker for unknown file", slog.String("content_type", detectedType))
+ tc := NewTextFileChecker()
+ mreader := io.MultiReader(bytes.NewReader(buffer), reader)
+ return tc.Run(mreader)
+ case strings.HasPrefix(detectedType, "image"):
+ slog.Debug("use image file checker for unknown file", slog.String("content_type", detectedType))
+ ic := NewImageFileChecker()
+ mreader := io.MultiReader(bytes.NewReader(buffer), reader)
+ return ic.Run(mreader)
+ case strings.HasPrefix(detectedType, "audio"):
+ slog.Debug("skip audio checker for unknown file", slog.String("content_type", detectedType))
+ return types.SensitiveCheckSkip, "skip binary audio file"
+ case strings.HasPrefix(detectedType, "video"):
+ slog.Debug("skip video checker for unknown file", slog.String("content_type", detectedType))
+ return types.SensitiveCheckSkip, "skip binary video file"
+ default:
+ slog.Debug("skip binary checker for unknown file", slog.String("content_type", detectedType))
+ return types.SensitiveCheckSkip, "skip binary file"
+ }
+
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/moderation/checker"
+)
+
+type repoComponentImpl struct {
+ checker sensitive.SensitiveChecker
+ rs database.RepoStore
+ rfs database.RepoFileStore
+ rfcs database.RepoFileCheckStore
+ git gitserver.GitServer
+}
+
+type RepoComponent interface {
+ UpdateRepoSensitiveCheckStatus(ctx context.Context, repoType types.RepositoryType, namespace string, name string, status types.SensitiveCheckStatus) error
+ CheckRepoFiles(ctx context.Context, repoType types.RepositoryType, namespace string, name string, options CheckOption) error
+ CheckRequestV2(ctx context.Context, req types.SensitiveRequestV2) (bool, error)
+}
+
+func NewRepoComponent(cfg *config.Config) (RepoComponent, error) {
+ c := &repoComponentImpl{checker: sensitive.NewAliyunGreenCheckerFromConfig(cfg)}
+ gs, err := git.NewGitServer(cfg)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server for sensitive component: %w", err)
+ }
+ c.rs = database.NewRepoStore()
+ c.rfs = database.NewRepoFileStore()
+ c.rfcs = database.NewRepoFileCheckStore()
+ c.git = gs
+
+ return c, nil
+}
+
+type CheckOption struct {
+ ForceCheck bool
+ // NotCheckFileExts []string
+ // ImageFileExts []string
+ LastRepoFileID int64
+ BatchSize int64
+ // MaxConcurrent int
+}
+
+func (c *repoComponentImpl) UpdateRepoSensitiveCheckStatus(ctx context.Context, repoType types.RepositoryType, namespace string, name string, status types.SensitiveCheckStatus) error {
+ repo, err := c.rs.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to get repo, error: %w", err)
+ }
+
+ repo.SensitiveCheckStatus = status
+ _, err = c.rs.UpdateRepo(ctx, *repo)
+ return err
+}
+
+func (c *repoComponentImpl) CheckRepoFiles(ctx context.Context, repoType types.RepositoryType, namespace string, name string, options CheckOption) error {
+ if options.BatchSize == 0 {
+ options.BatchSize = 10
+ }
+ // get repo id
+ repo, err := c.rs.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to get repo, error: %w", err)
+ }
+ repoID := repo.ID
+ for {
+ var files []*database.RepositoryFile
+ var err error
+ if options.ForceCheck {
+ files, err = c.rfs.BatchGet(ctx, repoID, options.LastRepoFileID, options.BatchSize)
+ } else {
+ files, err = c.rfs.BatchGetUnchcked(ctx, repoID, options.LastRepoFileID, options.BatchSize)
+ }
+
+ if err != nil {
+ return fmt.Errorf("failed to get repo files,repoID: %d, lastRepoFileID: %d, error: %w", repoID, options.LastRepoFileID, err)
+ }
+
+ for _, file := range files {
+ c.processFile(ctx, file)
+ }
+
+ count := len(files)
+ if count < int(options.BatchSize) {
+ break
+ }
+
+ options.LastRepoFileID = files[count-1].ID
+ }
+
+ return nil
+}
+
+func (c *repoComponentImpl) processFile(ctx context.Context, file *database.RepositoryFile) {
+ reader := NewRepoFileContentReader(file, c.git)
+ checker := checker.GetFileChecker(file.FileType, file.Path, file.LfsRelativePath)
+ status, msg := checker.Run(reader)
+ if status == types.SensitiveCheckException {
+ slog.Error("failed to check repo file content", slog.Int64("repo_id", file.RepositoryID), slog.Int64("repo_file_id", file.ID), slog.String("file", file.Path),
+ slog.String("err", msg))
+ }
+
+ err := c.saveCheckResult(ctx, file, status, msg)
+ if err != nil {
+ slog.Error("save check result failed", "error", err)
+ }
+}
+
+func (c *repoComponentImpl) saveCheckResult(ctx context.Context, file *database.RepositoryFile, status types.SensitiveCheckStatus, msg string) error {
+ fcr := &database.RepositoryFileCheck{
+ RepoFileID: file.ID,
+ Status: status,
+ Message: msg,
+ CreatedAt: time.Time{},
+ }
+
+ fail := status == types.SensitiveCheckFail
+ if fail {
+ // TODO: public a sensitive check status event to message queue
+ // set repository to private and set sensitive_check_status to fail
+ file.Repository.Private = true
+ file.Repository.SensitiveCheckStatus = types.SensitiveCheckFail
+ if _, err := c.rs.UpdateRepo(ctx, *file.Repository); err != nil {
+ slog.Error("failed to update repo sensitive check status", slog.Any("repository_id", file.Repository.ID),
+ slog.Int64("repository_file_id", file.ID), slog.String("path", file.Path),
+ slog.Any("error", err))
+ }
+ slog.Info("detect sensitive file content, set repository to private",
+ slog.String("path", file.Path), slog.Int64("repository_file_id", file.ID),
+ slog.Any("repository_id", file.Repository.ID))
+ }
+
+ // create repository file check record
+ err := c.rfcs.Upsert(ctx, *fcr)
+ if err != nil {
+ slog.Error("failed to create or update repository file check record", slog.Any("error", err),
+ slog.String("path", file.Path), slog.Int64("repo_file_id", file.ID))
+ }
+ return err
+}
+
+func (cc *repoComponentImpl) CheckRequestV2(ctx context.Context, req types.SensitiveRequestV2) (bool, error) {
+ fields := req.GetSensitiveFields()
+ for _, field := range fields {
+ pass, err := cc.checker.PassTextCheck(ctx, sensitive.Scenario(field.Scenario), field.Value())
+ if err != nil {
+ slog.Error("fail to check request sensitivity", slog.String("field", field.Name), slog.Any("error", err))
+ return false, fmt.Errorf("fail to check '%s' sensitivity, error: %w", field.Name, err)
+ }
+ if pass.IsSensitive {
+ slog.Error("found sensitive words in request", slog.String("field", field.Name))
+ return false, errors.New("found sensitive words in field: " + field.Name)
+ }
+ }
+ return true, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "sync"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type repoFileComponentImpl struct {
+ rfs database.RepoFileStore
+ rs database.RepoStore
+ gs gitserver.GitServer
+}
+
+type RepoFileComponent interface {
+ GenRepoFileRecords(ctx context.Context, repoType types.RepositoryType, namespace, name string) error
+ GenRepoFileRecordsBatch(ctx context.Context, repoType types.RepositoryType, lastRepoID int64, concurrency int) error
+ DetectRepoSensitiveCheckStatus(ctx context.Context, repoType types.RepositoryType, namespace, name string) error
+}
+
+func NewRepoFileComponent(conf *config.Config) (RepoFileComponent, error) {
+ c := &repoFileComponentImpl{
+ rfs: database.NewRepoFileStore(),
+ rs: database.NewRepoStore(),
+ }
+ gs, err := git.NewGitServer(conf)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server, error: %w", err)
+ }
+
+ c.gs = gs
+ return c, nil
+}
+func (c *repoFileComponentImpl) GenRepoFileRecords(ctx context.Context, repoType types.RepositoryType, namespace, name string) error {
+ repo, err := c.rs.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ return c.createRepoFileRecords(ctx, *repo, "", c.gs.GetRepoFileTree)
+}
+
+func (c *repoFileComponentImpl) GenRepoFileRecordsBatch(ctx context.Context, repoType types.RepositoryType, lastRepoID int64, concurrency int) error {
+ tokens := make(chan struct{}, concurrency)
+ for i := 0; i < concurrency; i++ {
+ tokens <- struct{}{}
+ }
+ wg := &sync.WaitGroup{}
+ //TODO: load last repo id from redis cache
+ batch := 10
+ for {
+ repos, err := c.rs.BatchGet(ctx, repoType, lastRepoID, batch)
+ if err != nil {
+ return fmt.Errorf("failed to get repos in batch, error: %w", err)
+ }
+ for _, repo := range repos {
+ //wait
+ <-tokens
+ wg.Add(1)
+ go func(repo database.Repository) {
+ slog.Info("start to get files of repository", slog.Any("repoType", repoType), slog.String("path", repo.Path))
+ //get file paths of repo
+ err := c.createRepoFileRecords(ctx, repo, "", c.gs.GetRepoFileTree)
+ if err != nil {
+ slog.Error("fail to get all files of repository",
+ slog.String("path", repo.Path), slog.String("repo_type", string(repo.RepositoryType)),
+ slog.String("error", err.Error()))
+ }
+ tokens <- struct{}{}
+ wg.Done()
+ }(repo)
+
+ }
+
+ if len(repos) < batch {
+ break
+ }
+ lastRepoID = repos[len(repos)-1].ID
+ }
+
+ wg.Wait()
+ return nil
+}
+
+func (c *repoFileComponentImpl) createRepoFileRecords(ctx context.Context, repo database.Repository, folder string, gsTree func(ctx context.Context, req gitserver.GetRepoInfoByPathReq) ([]*types.File, error)) error {
+ namespace, name := repo.NamespaceAndName()
+ var files []*types.File
+
+ getRepoFileTree := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: name,
+ Ref: repo.DefaultBranch,
+ Path: folder,
+ RepoType: repo.RepositoryType,
+ }
+ gitFiles, err := gsTree(context.Background(), getRepoFileTree)
+ if err != nil {
+ return fmt.Errorf("failed to get repo file tree,%w", err)
+ }
+ for _, file := range gitFiles {
+ if file.Type == "dir" {
+ err := c.createRepoFileRecords(ctx, repo, file.Path, gsTree)
+ if err != nil {
+ return err
+ }
+ } else {
+ files = append(files, file)
+ }
+ }
+ //get all files
+ for _, file := range files {
+ // save repo files into db
+ rf := database.RepositoryFile{
+ RepositoryID: repo.ID,
+ Path: file.Path,
+ FileType: file.Type,
+ Size: file.Size,
+ CommitSha: file.SHA,
+ LfsRelativePath: file.LfsRelativePath,
+ Branch: repo.DefaultBranch,
+ }
+
+ var exists bool
+ var err error
+ if exists, err = c.rfs.Exists(ctx, rf); err != nil {
+ slog.Error("failed to check repository file exists", slog.Any("repo_id", repo.ID),
+ slog.String("file_path", rf.Path), slog.String("error", err.Error()))
+ continue
+ }
+
+ if exists {
+ slog.Info("skip create exist repository file", slog.Any("repo_id", repo.ID), slog.String("file_path", rf.Path))
+ continue
+ }
+ if err := c.rfs.Create(ctx, &rf); err != nil {
+ slog.Error("failed to save repository file", slog.Any("repo_id", repo.ID),
+ slog.String("error", err.Error()))
+ return fmt.Errorf("failed to save repository file, error: %w", err)
+ }
+ }
+ return nil
+}
+
+func (c *repoFileComponentImpl) DetectRepoSensitiveCheckStatus(ctx context.Context, repoType types.RepositoryType, namespace, name string) error {
+ repo, err := c.rs.FindByPath(ctx, repoType, namespace, name)
+ if err != nil {
+ return fmt.Errorf("failed to find repo, error: %w", err)
+ }
+ //TODO:handler other branches
+ branch := repo.DefaultBranch
+
+ status := types.SensitiveCheckFail
+ exists, err := c.rfs.ExistsSensitiveCheckRecord(ctx, repo.ID, branch, types.SensitiveCheckFail)
+ if err != nil {
+ return fmt.Errorf("failed to check repo file sensitive check record exists, error: %w", err)
+ }
+ if exists {
+ repo.SensitiveCheckStatus = status
+ _, err = c.rs.UpdateRepo(ctx, *repo)
+ return err
+ }
+
+ status = types.SensitiveCheckException
+ exists, err = c.rfs.ExistsSensitiveCheckRecord(ctx, repo.ID, branch, types.SensitiveCheckException)
+ if err != nil {
+ return fmt.Errorf("failed to check repo file sensitive check record exists, error: %w", err)
+ }
+ if exists {
+ repo.SensitiveCheckStatus = status
+ _, err = c.rs.UpdateRepo(ctx, *repo)
+ return err
+ }
+
+ repo.SensitiveCheckStatus = types.SensitiveCheckPass
+ _, err = c.rs.UpdateRepo(ctx, *repo)
+ return err
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "io"
+ "log/slog"
+ "sync"
+
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+)
+
+type RepoFileContentReader struct {
+ file *database.RepositoryFile
+ git gitserver.GitServer
+ innerReader io.ReadCloser
+ once *sync.Once
+}
+
+var _ io.ReadCloser = (*RepoFileContentReader)(nil)
+
+func NewRepoFileContentReader(file *database.RepositoryFile, git gitserver.GitServer) *RepoFileContentReader {
+ return &RepoFileContentReader{
+ file: file,
+ git: git,
+ once: &sync.Once{},
+ }
+}
+
+func (c *RepoFileContentReader) Read(p []byte) (n int, err error) {
+ c.lazyInit()
+
+ if c.innerReader == nil {
+ return 0, errors.New("failed to read file content as git file reader not initialized")
+ }
+ return c.innerReader.Read(p)
+}
+
+func (c *RepoFileContentReader) Close() error {
+ if c.innerReader == nil {
+ return errors.New("failed to close reader as git file reader not initialized")
+ }
+ return c.innerReader.Close()
+}
+
+func (c *RepoFileContentReader) lazyInit() {
+ c.once.Do(func() {
+ namespace, name := c.file.Repository.NamespaceAndName()
+ req := gitserver.GetRepoInfoByPathReq{
+ Namespace: namespace,
+ Name: name,
+ Path: c.file.Path,
+ RepoType: c.file.Repository.RepositoryType,
+ Ref: c.file.Repository.DefaultBranch,
+ }
+
+ ctx := context.Background()
+ var err error
+ c.innerReader, _, err = c.git.GetRepoFileReader(ctx, req)
+ if err != nil {
+ slog.Error("failed to create git file reader", slog.Any("error", err), slog.String("path", c.file.Path), slog.Int64("repository_file_id", c.file.ID))
+ }
+ })
+}
+
+
+
package handler
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "go.temporal.io/sdk/client"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/moderation/component"
+ "opencsg.com/csghub-server/moderation/workflow"
+ "opencsg.com/csghub-server/moderation/workflow/common"
+)
+
+type RepoHandler struct {
+ rc component.RepoComponent
+ config *config.Config
+}
+
+func NewRepoHandler(config *config.Config) (*RepoHandler, error) {
+ c, err := component.NewRepoComponent(config)
+ if err != nil {
+ return nil, err
+ }
+
+ return &RepoHandler{
+ rc: c,
+ config: config,
+ }, nil
+}
+
+func (h *RepoHandler) FullCheck(c *gin.Context) {
+ type request struct {
+ Namespace string `json:"namespace"`
+ Name string `json:"name"`
+ RepoType types.RepositoryType `json:"repo_type"`
+ }
+
+ var req request
+ // binding request and check error
+ if err := c.BindJSON(&req); err != nil {
+ slog.Error("invalid request for full check", slog.Any("error", err))
+ httpbase.BadRequest(c, err.Error())
+ return
+ }
+
+ //start workflow to do full check
+ workflowClient := workflow.GetWorkflowClient()
+ workflowOptions := client.StartWorkflowOptions{
+ TaskQueue: "moderation_repo_full_check_queue",
+ }
+
+ we, err := workflowClient.ExecuteWorkflow(context.Background(), workflowOptions, workflow.RepoFullCheckWorkflow,
+ common.Repo{
+ Namespace: req.Namespace,
+ Name: req.Name,
+ RepoType: req.RepoType,
+ }, h.config)
+ if err != nil {
+ httpbase.ServerError(c, fmt.Errorf("failed to start repo full check workflow, error: %w", err))
+ return
+ }
+
+ slog.Info("start repo full check workflow", slog.String("workflow_id", we.GetID()))
+ httpbase.OK(c, nil)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/sensitive"
+ "opencsg.com/csghub-server/common/config"
+)
+
+type SensitiveHandler struct {
+ c sensitive.SensitiveChecker
+}
+
+func NewSensitiveHandler(cfg *config.Config) (*SensitiveHandler, error) {
+ return &SensitiveHandler{
+ c: sensitive.NewAliyunGreenCheckerFromConfig(cfg),
+ }, nil
+}
+
+func (h *SensitiveHandler) Text(ctx *gin.Context) {
+ type req struct {
+ Scenario sensitive.Scenario `json:"scenario"`
+ Text string `json:"text"`
+ }
+ var (
+ r req
+ err error
+ )
+ if err = ctx.ShouldBindJSON(&r); err != nil {
+ slog.Error("Bad request format", slog.String("err", err.Error()))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ result, err := h.c.PassTextCheck(ctx, r.Scenario, r.Text)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, result)
+
+}
+
+func (h *SensitiveHandler) Image(ctx *gin.Context) {
+ type req struct {
+ Scenario sensitive.Scenario `json:"scenario"`
+ OssBucketName string `json:"oss_bucket_name"`
+ OssObjectName string `json:"oss_object_name"`
+ }
+ var (
+ r req
+ err error
+ )
+ if err = ctx.ShouldBindJSON(&r); err != nil {
+ slog.Error("Bad request format", slog.String("err", err.Error()))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ result, err := h.c.PassImageCheck(ctx, r.Scenario, r.OssBucketName, r.OssObjectName)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, result)
+}
+
+
+
package router
+
+import (
+ "fmt"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/moderation/handler"
+)
+
+func NewRouter(config *config.Config) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+ // r.Use(middleware.Authenticator(config))
+
+ apiV1Group := r.Group("/api/v1")
+ {
+ apiV1Group.GET("/healthz", healthz)
+ }
+
+ mc, err := handler.NewRepoHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating repo handler:%w", err)
+ }
+ apiV1Group.POST("/repo", mc.FullCheck)
+ sc, err := handler.NewSensitiveHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive handler:%w", err)
+ }
+ apiV1Group.POST("/text", sc.Text)
+ apiV1Group.POST("/image", sc.Image)
+
+ return r, nil
+}
+
+func healthz(ctx *gin.Context) {
+ ctx.Writer.WriteHeader(http.StatusOK)
+}
+
+
+
package activity
+
+import (
+ "context"
+ "fmt"
+
+ "go.temporal.io/sdk/activity"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/moderation/component"
+ "opencsg.com/csghub-server/moderation/workflow/common"
+)
+
+// GenRepoFileList generates repository file records for a given repository.
+// This function is an activity that can be used in a workflow.
+func GenRepoFileList(ctx context.Context, req common.Repo, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("gen repo files start", "repo", req)
+ rfc, err := component.NewRepoFileComponent(config)
+ if err != nil {
+ return fmt.Errorf("failed to create repo file component, error: %w", err)
+ }
+ return rfc.GenRepoFileRecords(ctx, req.RepoType, req.Namespace, req.Name)
+}
+
+func CheckRepoFiles(ctx context.Context, req common.Repo, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("check repo files start", "repoType", req.RepoType, "namespace", req.Namespace, "name", req.Name)
+ rc, err := component.NewRepoComponent(config)
+ if err != nil {
+ return fmt.Errorf("failed to create repo component, error: %w", err)
+ }
+
+ err = rc.CheckRepoFiles(ctx, req.RepoType, req.Namespace, req.Name, component.CheckOption{})
+ if err != nil {
+ logger.Error("check repo files failed", "error", err, "repoType", req.RepoType, "namespace", req.Namespace, "name", req.Name)
+ return err
+ }
+ logger.Info("check repo files complete", "repoType", req.RepoType, "namespace", req.Namespace, "name", req.Name)
+ return nil
+}
+
+// RepoSensitiveCheckPending updates the sensitive check status of a repository to pending.
+// This function is an activity that can be used in a workflow.
+func RepoSensitiveCheckPending(ctx context.Context, req common.Repo, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("set repo sensitive check status to pending start", "repoType", req.RepoType, "namespace", req.Namespace, "name", req.Name)
+ rc, err := component.NewRepoComponent(config)
+ if err != nil {
+ return fmt.Errorf("failed to create repo component, error: %w", err)
+ }
+
+ return rc.UpdateRepoSensitiveCheckStatus(ctx, req.RepoType, req.Namespace, req.Name, types.SensitiveCheckPending)
+}
+
+func DetectRepoSensitiveCheckStatus(ctx context.Context, req common.Repo, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("detect repo sensitive check status start", "repoType", req.RepoType, "namespace", req.Namespace, "name", req.Name)
+ rfc, err := component.NewRepoFileComponent(config)
+ if err != nil {
+ return fmt.Errorf("failed to create repo component, error: %w", err)
+ }
+
+ return rfc.DetectRepoSensitiveCheckStatus(ctx, req.RepoType, req.Namespace, req.Name)
+}
+
+
+
package workflow
+
+import (
+ "time"
+
+ "go.temporal.io/sdk/temporal"
+ "go.temporal.io/sdk/workflow"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/moderation/workflow/activity"
+ "opencsg.com/csghub-server/moderation/workflow/common"
+)
+
+func RepoFullCheckWorkflow(ctx workflow.Context, repo common.Repo, config *config.Config) error {
+ logger := workflow.GetLogger(ctx)
+
+ retryPolicy := &temporal.RetryPolicy{
+ // InitialInterval: time.Second,
+ // BackoffCoefficient: 2.0,
+ // MaximumInterval: time.Minute,
+ MaximumAttempts: 3,
+ }
+
+ options := workflow.ActivityOptions{
+ // Timeout options specify when to automatically timeout Activity functions.
+ StartToCloseTimeout: 24 * time.Hour,
+ // Optionally provide a customized RetryPolicy.
+ RetryPolicy: retryPolicy,
+ }
+ ctx = workflow.WithActivityOptions(ctx, options)
+ var err error
+
+ err = workflow.ExecuteActivity(ctx, activity.RepoSensitiveCheckPending, repo, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to update repo sensitive check status", "error", err, "repo", repo, "status", types.SensitiveCheckPending)
+ return err
+ }
+
+ // 1. generate repo file list
+ err = workflow.ExecuteActivity(ctx, activity.GenRepoFileList, repo, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to generate repo file list", "error", err, "repo", repo)
+ return err
+ }
+ // 2. check repo file content
+ err = workflow.ExecuteActivity(ctx, activity.CheckRepoFiles, repo, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to check repo files", "error", err, "repo", repo)
+ return err
+ }
+ // 3. update repo sensitive check status
+ err = workflow.ExecuteActivity(ctx, activity.DetectRepoSensitiveCheckStatus, repo, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to detect repo sensitive check status", "error", err, "repo", repo)
+ return err
+ }
+
+ return nil
+}
+
+
+
package workflow
+
+import (
+ "fmt"
+
+ "go.temporal.io/sdk/client"
+ "go.temporal.io/sdk/worker"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/moderation/workflow/activity"
+)
+
+var wfClient client.Client
+var wfWorker worker.Worker
+
+func StartWorker(config *config.Config) error {
+ var err error
+ wfClient, err = client.Dial(client.Options{
+ HostPort: config.WorkFLow.Endpoint,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to create workflow client, error:%w", err)
+ }
+
+ wfWorker = worker.New(wfClient, "moderation_repo_full_check_queue", worker.Options{})
+ wfWorker.RegisterWorkflow(RepoFullCheckWorkflow)
+ wfWorker.RegisterActivity(activity.RepoSensitiveCheckPending)
+ wfWorker.RegisterActivity(activity.GenRepoFileList)
+ wfWorker.RegisterActivity(activity.CheckRepoFiles)
+ wfWorker.RegisterActivity(activity.DetectRepoSensitiveCheckStatus)
+
+ return wfWorker.Start()
+}
+
+func StopWorker() {
+ if wfWorker != nil {
+ wfWorker.Stop()
+ }
+ if wfClient != nil {
+ wfClient.Close()
+ }
+}
+
+func GetWorkflowClient() client.Client {
+ return wfClient
+}
+
+
+
package mq
+
+import (
+ "opencsg.com/csghub-server/common/config"
+)
+
+func Init(config *config.Config) (MessageQueue, error) {
+ sysMQ, err := NewNats(config)
+ if err != nil {
+ return nil, err
+ }
+ err = sysMQ.GetJetStream()
+ if err != nil {
+ return nil, err
+ }
+ return sysMQ, nil
+}
+
+
+
package mq
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/nats-io/nats.go"
+ "github.com/nats-io/nats.go/jetstream"
+ "opencsg.com/csghub-server/common/config"
+)
+
+var _ MessageQueue = (*NatsHandler)(nil)
+
+type EventConfig struct {
+ StreamName string // stream name of event
+ ConsumerName string // durable consumer name
+}
+
+type DLQEventConfig struct {
+ EventConfig
+ FeeSubjectName string // subject of fee dlq
+ MeterSubjectName string // subject of meter dlq
+ RechargeSubjectName string // subject of recharge dlq
+}
+
+type RequestSubject struct {
+ fee string // fee charging subject
+ token string // token subject
+ quota string // quota subject
+ nobalance string // subject name for pub notification of no balance
+ duration string // duration subject
+ orderExpired string // subject name for pub notification of order expired
+ rechargeSucceed string //
+}
+
+var (
+ nats_connect_timeout time.Duration = 2 * time.Second // second
+ nats_reconnect_wait time.Duration = 10 * time.Second // second
+
+ feeCfg = EventConfig{
+ StreamName: "accountingEventStream", // fee request
+ ConsumerName: "accountingServerDurableConsumer",
+ }
+
+ notifyCfg = EventConfig{
+ StreamName: "accountingNotifyStream", // notify message
+ ConsumerName: "accountingNotifyDurableConsumer",
+ }
+
+ dlqCfg = EventConfig{
+ StreamName: "accountingDlqStream", // DLQ
+ ConsumerName: "accountingDlqDurableConsumer",
+ }
+
+ dlq = DLQEventConfig{
+ EventConfig: dlqCfg,
+ FeeSubjectName: "accounting.dlq.fee",
+ MeterSubjectName: "accounting.dlq.meter",
+ RechargeSubjectName: "accounting.dlq.recharge",
+ }
+
+ meterCfg = EventConfig{
+ StreamName: "meteringEventStream", // metering request
+ ConsumerName: "metertingServerDurableConsumer",
+ }
+
+ orderCfg = EventConfig{
+ StreamName: "accountingOrderStream", // order
+ ConsumerName: "accountingOrderDurableConsumer",
+ }
+
+ rechargeCfg = EventConfig{
+ StreamName: "rechargeStream",
+ ConsumerName: "rechargeDurableConsumer",
+ }
+)
+
+type NatsHandler struct {
+ conn *nats.Conn
+
+ msgFetchTimeoutInSec int
+ feeReqSub RequestSubject
+ meterReqSub RequestSubject
+ orderReqSub RequestSubject
+ rechargeReqSub RequestSubject
+ feeEvtCfg jetstream.StreamConfig
+ feeConsumerCfg jetstream.ConsumerConfig
+ notifyEvtCfg jetstream.StreamConfig
+ notifyConsumerCfg jetstream.ConsumerConfig
+ dlqEvtCfg jetstream.StreamConfig
+ meterEvtCfg jetstream.StreamConfig
+ meterConsumerCfg jetstream.ConsumerConfig
+ orderEvtCfg jetstream.StreamConfig
+ orderConsumerCfg jetstream.ConsumerConfig
+ rechargeEvtCfg jetstream.StreamConfig
+ rechargeConsumerCfg jetstream.ConsumerConfig
+
+ js jetstream.JetStream
+ feeJsc jetstream.Consumer
+ meterJsc jetstream.Consumer
+ orderJsc jetstream.Consumer
+ rechargeJsc jetstream.Consumer
+}
+
+func initStreamAndConsumerConfig(cfg EventConfig, subjectNames []string) (jetstream.StreamConfig, jetstream.ConsumerConfig) {
+ return jetstream.StreamConfig{
+ Name: cfg.StreamName, Subjects: subjectNames,
+ MaxConsumers: -1, MaxMsgs: -1, MaxBytes: -1,
+ },
+ jetstream.ConsumerConfig{
+ Name: cfg.ConsumerName, Durable: cfg.ConsumerName, FilterSubject: subjectNames[0],
+ AckPolicy: jetstream.AckExplicitPolicy, DeliverPolicy: jetstream.DeliverAllPolicy,
+ }
+}
+
+func NewNats(config *config.Config) (*NatsHandler, error) {
+ nc, err := nats.Connect(
+ config.Nats.URL,
+ nats.Timeout(nats_connect_timeout),
+ nats.ReconnectWait(nats_reconnect_wait),
+ nats.MaxReconnects(-1),
+ )
+ if err != nil {
+ return nil, err
+ }
+ feeEC, feeCC := initStreamAndConsumerConfig(feeCfg, []string{config.Nats.FeeRequestSubject})
+ notifyEC, notifyCC := initStreamAndConsumerConfig(notifyCfg, []string{config.Nats.FeeNotifyNoBalanceSubject})
+ dlqEC, _ := initStreamAndConsumerConfig(dlqCfg, []string{dlq.FeeSubjectName, dlq.MeterSubjectName, dlq.RechargeSubjectName})
+ meterEC, meterCC := initStreamAndConsumerConfig(meterCfg, []string{config.Nats.MeterRequestSubject})
+ orderEC, orderCC := initStreamAndConsumerConfig(orderCfg, []string{config.Nats.OrderExpiredSubject})
+ rechargeEC, rechargeCC := initStreamAndConsumerConfig(rechargeCfg, []string{config.Nats.RechargeSucceedSubject})
+
+ return &NatsHandler{
+ conn: nc,
+ msgFetchTimeoutInSec: config.Nats.MsgFetchTimeoutInSEC,
+ feeReqSub: RequestSubject{
+ fee: config.Nats.FeeSendSubject,
+ token: config.Nats.TokenSendSubject,
+ quota: config.Nats.QuotaSendSubject,
+ nobalance: config.Nats.FeeNotifyNoBalanceSubject,
+ },
+ meterReqSub: RequestSubject{
+ duration: config.Nats.MeterDurationSendSubject,
+ token: config.Nats.MeterTokenSendSubject,
+ quota: config.Nats.MeterQuotaSendSubject,
+ },
+ orderReqSub: RequestSubject{
+ orderExpired: config.Nats.OrderExpiredSubject,
+ },
+ rechargeReqSub: RequestSubject{
+ rechargeSucceed: config.Nats.RechargeSucceedSubject,
+ },
+ feeEvtCfg: feeEC,
+ feeConsumerCfg: feeCC,
+ notifyEvtCfg: notifyEC,
+ notifyConsumerCfg: notifyCC,
+ dlqEvtCfg: dlqEC,
+ meterEvtCfg: meterEC,
+ meterConsumerCfg: meterCC,
+ orderEvtCfg: orderEC,
+ orderConsumerCfg: orderCC,
+ rechargeEvtCfg: rechargeEC,
+ rechargeConsumerCfg: rechargeCC,
+ }, nil
+}
+
+func (nh *NatsHandler) GetConn() *nats.Conn {
+ return nh.conn
+}
+
+func (nh *NatsHandler) GetJetStream() error {
+ js, err := jetstream.New(nh.conn)
+ if err != nil {
+ return err
+ }
+ nh.js = js
+ return nil
+}
+
+func (nh *NatsHandler) CreateOrUpdateStream(ctx context.Context, streamName string, streamCfg jetstream.StreamConfig) (jetstream.Stream, error) {
+ err := nh.VerifyStreamByName(streamName)
+ if err != nil && !errors.Is(err, jetstream.ErrStreamNotFound) {
+ return nil, err
+ }
+ jss, err := nh.js.CreateOrUpdateStream(ctx, streamCfg)
+ if err != nil {
+ return nil, err
+ }
+ return jss, err
+}
+
+func (nh *NatsHandler) BuildEventStreamAndConsumer(cfg EventConfig, streamCfg jetstream.StreamConfig, consumerCfg jetstream.ConsumerConfig) (jetstream.Consumer, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+
+ err := nh.GetJetStream()
+ if err != nil {
+ return nil, err
+ }
+
+ jss, err := nh.CreateOrUpdateStream(ctx, cfg.StreamName, streamCfg)
+ if err != nil {
+ return nil, err
+ }
+
+ jsc, err := jss.CreateOrUpdateConsumer(ctx, consumerCfg)
+ if err != nil {
+ return nil, err
+ }
+ return jsc, nil
+}
+
+func (nh *NatsHandler) BuildFeeEventStream() error {
+ jsc, err := nh.BuildEventStreamAndConsumer(feeCfg, nh.feeEvtCfg, nh.feeConsumerCfg)
+ if err != nil {
+ return err
+ }
+ nh.feeJsc = jsc
+ return nil
+}
+
+func (nh *NatsHandler) BuildMeterEventStream() error {
+ jsc, err := nh.BuildEventStreamAndConsumer(meterCfg, nh.meterEvtCfg, nh.meterConsumerCfg)
+ if err != nil {
+ return err
+ }
+ nh.meterJsc = jsc
+ return nil
+}
+
+func (nh *NatsHandler) BuildRechargeEventStream() error {
+ jsc, err := nh.BuildEventStreamAndConsumer(rechargeCfg, nh.rechargeEvtCfg, nh.rechargeConsumerCfg)
+ if err != nil {
+ return err
+ }
+ nh.rechargeJsc = jsc
+ return nil
+}
+
+func (nh *NatsHandler) BuildOrderEventStream() error {
+ jsc, err := nh.BuildEventStreamAndConsumer(orderCfg, nh.orderEvtCfg, nh.orderConsumerCfg)
+ if err != nil {
+ return err
+ }
+ nh.orderJsc = jsc
+ return nil
+}
+
+func (nh *NatsHandler) BuildNotifyStream() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ err := nh.GetJetStream()
+ if err != nil {
+ return err
+ }
+ _, err = nh.CreateOrUpdateStream(ctx, notifyCfg.StreamName, nh.notifyEvtCfg)
+ return err
+}
+
+func (nh *NatsHandler) BuildDLQStream() error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ err := nh.GetJetStream()
+ if err != nil {
+ return err
+ }
+ _, err = nh.CreateOrUpdateStream(ctx, dlqCfg.StreamName, nh.dlqEvtCfg)
+ return err
+}
+
+func (nh *NatsHandler) FetchFeeEventMessages(batch int) (jetstream.MessageBatch, error) {
+ msgs, err := nh.feeJsc.Fetch(batch, jetstream.FetchMaxWait(time.Duration(nh.msgFetchTimeoutInSec)*time.Second))
+ return msgs, err
+}
+
+func (nh *NatsHandler) FetchMeterEventMessages(batch int) (jetstream.MessageBatch, error) {
+ msgs, err := nh.meterJsc.Fetch(batch, jetstream.FetchMaxWait(time.Duration(nh.msgFetchTimeoutInSec)*time.Second))
+ return msgs, err
+}
+
+func (nh *NatsHandler) FetchRechargeEventMessages(batch int) (jetstream.MessageBatch, error) {
+ msgs, err := nh.rechargeJsc.Fetch(batch, jetstream.FetchMaxWait(time.Duration(nh.msgFetchTimeoutInSec)*time.Second))
+ return msgs, err
+}
+
+func (nh *NatsHandler) VerifyStreamByName(streamName string) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ _, err := nh.js.Stream(ctx, streamName)
+ return err
+}
+
+func (nh *NatsHandler) VerifyFeeEventStream() error {
+ return nh.VerifyStreamByName(feeCfg.StreamName)
+}
+
+func (nh *NatsHandler) VerifyMeteringStream() error {
+ return nh.VerifyStreamByName(meterCfg.StreamName)
+}
+
+func (nh *NatsHandler) VerifyRechargeStream() error {
+ return nh.VerifyStreamByName(rechargeCfg.StreamName)
+}
+
+func (nh *NatsHandler) VerifyNotifyStream() error {
+ return nh.VerifyStreamByName(notifyCfg.StreamName)
+}
+
+func (nh *NatsHandler) VerifyDLQStream() error {
+ return nh.VerifyStreamByName(dlqCfg.StreamName)
+}
+
+func (nh *NatsHandler) PublishData(subject string, data []byte) error {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ _, err := nh.js.Publish(ctx, subject, data)
+ return err
+}
+
+func (nh *NatsHandler) PublishNotificationForNoBalance(data []byte) error {
+ return nh.PublishData(nh.feeReqSub.nobalance, data)
+}
+
+func (nh *NatsHandler) PublishFeeCreditData(data []byte) error {
+ return nh.PublishData(nh.feeReqSub.fee, data)
+}
+
+func (nh *NatsHandler) PublishFeeTokenData(data []byte) error {
+ return nh.PublishData(nh.feeReqSub.token, data)
+}
+
+func (nh *NatsHandler) PublishFeeQuotaData(data []byte) error {
+ return nh.PublishData(nh.feeReqSub.quota, data)
+}
+
+func (nh *NatsHandler) PublishFeeDataToDLQ(data []byte) error {
+ return nh.PublishData(dlq.FeeSubjectName, data)
+}
+
+func (nh *NatsHandler) PublishMeterDataToDLQ(data []byte) error {
+ return nh.PublishData(dlq.MeterSubjectName, data)
+}
+
+func (nh *NatsHandler) PublishRechargeDataToDLQ(data []byte) error {
+ return nh.PublishData(dlq.RechargeSubjectName, data)
+}
+
+func (nh *NatsHandler) PublishMeterDurationData(data []byte) error {
+ return nh.PublishData(nh.meterReqSub.duration, data)
+}
+
+func (nh *NatsHandler) PublishRechargeDurationData(data []byte) error {
+ return nh.PublishData(nh.rechargeReqSub.rechargeSucceed, data)
+}
+
+func (nh *NatsHandler) PublishOrderExpiredData(data []byte) error {
+ return nh.PublishData(nh.orderReqSub.orderExpired, data)
+}
+
+func (nh *NatsHandler) BuildNotifyConsumerWithName(consumerName string) (jetstream.Consumer, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ ec := EventConfig{StreamName: notifyCfg.StreamName, ConsumerName: consumerName}
+ _, conCfg := initStreamAndConsumerConfig(ec, []string{nh.feeReqSub.nobalance})
+ consumer, err := nh.js.CreateOrUpdateConsumer(ctx, notifyCfg.StreamName, conCfg)
+ return consumer, err
+}
+
+func (nh *NatsHandler) BuildOrderConsumerWithName(consumerName string) (jetstream.Consumer, error) {
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ ec := EventConfig{StreamName: orderCfg.StreamName, ConsumerName: consumerName}
+ _, conCfg := initStreamAndConsumerConfig(ec, []string{nh.orderReqSub.orderExpired})
+ consumer, err := nh.js.CreateOrUpdateConsumer(ctx, orderCfg.StreamName, conCfg)
+ return consumer, err
+}
+
+
+
package accounting
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "time"
+)
+
+type SyncQuotaStatement struct {
+ ID int64 `json:"id"`
+ UserID int64 `json:"user_id"`
+ RepoPath string `json:"repo_path"`
+ RepoType string `json:"repo_type"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+type SyncQuotaStatementRes struct {
+ Message string `json:"msg"`
+ Data SyncQuotaStatement `json:"data"`
+}
+
+type GetSyncQuotaStatementsReq struct {
+ RepoPath string `json:"repo_path"`
+ RepoType string `json:"repo_type"`
+ AccessToken string `json:"-"`
+}
+
+type CreateSyncQuotaStatementReq = GetSyncQuotaStatementsReq
+
+func (c *AccountingClient) CreateSyncQuotaStatement(opt *CreateSyncQuotaStatementReq) (*Response, error) {
+ header := http.Header{"content-type": []string{"application/json"}}
+ body, err := json.Marshal(&opt)
+ if err != nil {
+ return nil, err
+ }
+ if opt.AccessToken != "" {
+ header.Add("Authorization", "Bearer "+opt.AccessToken)
+ }
+ _, resp, err := c.getResponse("POST", "/accounting/multisync/downloads", header, bytes.NewReader(body))
+ return resp, err
+}
+
+func (c *AccountingClient) GetSyncQuotaStatement(opt *GetSyncQuotaStatementsReq) (*SyncQuotaStatement, *Response, error) {
+ s := new(SyncQuotaStatementRes)
+ header := http.Header{}
+ if opt.AccessToken != "" {
+ header.Add("Authorization", "Bearer "+opt.AccessToken)
+ }
+ resp, err := c.getParsedResponse("GET", fmt.Sprintf("/accounting/multisync/download?repo_path=%s&repo_type=%s", opt.RepoPath, opt.RepoType), header, nil, s)
+ return &s.Data, resp, err
+}
+
+
+
package accounting
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "sync"
+ "time"
+
+ "opencsg.com/csghub-server/common/config"
+)
+
+type AccountingClient struct {
+ baseURL string
+ httpClient *http.Client
+ mutex sync.RWMutex
+ ctx context.Context
+}
+
+type Response struct {
+ *http.Response
+}
+
+func NewAccountingClient(config *config.Config) (*AccountingClient, error) {
+ if config.Accounting.Host == "" {
+ return nil, fmt.Errorf("accounting host should be configured")
+ }
+
+ if config.Accounting.Port == 0 {
+ return nil, fmt.Errorf("accounting port should be configured")
+ }
+ if config.APIToken == "" {
+ return nil, fmt.Errorf("api token should be configured")
+ }
+
+ return &AccountingClient{
+ baseURL: fmt.Sprintf("%s:%d", config.Accounting.Host, config.Accounting.Port),
+ httpClient: &http.Client{
+ Timeout: time.Second * 5,
+ },
+ ctx: context.Background(),
+ }, nil
+}
+
+func (c *AccountingClient) getParsedResponse(method, path string, header http.Header, body io.Reader, obj interface{}) (*Response, error) {
+ data, resp, err := c.getResponse(method, path, header, body)
+ if err != nil {
+ return resp, err
+ }
+ return resp, json.Unmarshal(data, obj)
+}
+
+func (c *AccountingClient) getResponse(method, path string, header http.Header, body io.Reader) ([]byte, *Response, error) {
+ resp, err := c.doRequest(method, path, header, body)
+ if err != nil {
+ return nil, resp, err
+ }
+ defer resp.Body.Close()
+
+ // check for errors
+ data, err := statusCodeToErr(resp)
+ if err != nil {
+ return data, resp, err
+ }
+
+ // success (2XX), read body
+ data, err = io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, resp, err
+ }
+
+ return data, resp, nil
+}
+
+// Converts a response for a HTTP status code indicating an error condition
+// (non-2XX) to a well-known error value and response body. For non-problematic
+// (2XX) status codes nil will be returned. Note that on a non-2XX response, the
+// response body stream will have been read and, hence, is closed on return.
+func statusCodeToErr(resp *Response) (body []byte, err error) {
+ // no error
+ if resp.StatusCode/100 == 2 {
+ return nil, nil
+ }
+
+ //
+ // error: body will be read for details
+ //
+ defer resp.Body.Close()
+ data, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("body read on HTTP error %d: %v", resp.StatusCode, err)
+ }
+
+ // Try to unmarshal and get an error message
+ errMap := make(map[string]interface{})
+ if err = json.Unmarshal(data, &errMap); err != nil {
+ // when the JSON can't be parsed, data was probably empty or a
+ // plain string, so we try to return a helpful error anyway
+ path := resp.Request.URL.Path
+ method := resp.Request.Method
+ header := resp.Request.Header
+ return data, fmt.Errorf("unknown API Error: %d\nRequest: '%s' with '%s' method '%s' header and '%s' body", resp.StatusCode, path, method, header, string(data))
+ }
+
+ if msg, ok := errMap["message"]; ok {
+ return data, fmt.Errorf("%v", msg)
+ }
+
+ // If no error message, at least give status and data
+ return data, fmt.Errorf("%s: %s", resp.Status, string(data))
+}
+
+func (c *AccountingClient) doRequest(method, path string, header http.Header, body io.Reader) (*Response, error) {
+ c.mutex.RLock()
+ req, err := http.NewRequestWithContext(c.ctx, method, c.baseURL+"/api/v1"+path, body)
+ if err != nil {
+ c.mutex.RUnlock()
+ return nil, err
+ }
+
+ client := c.httpClient
+ c.mutex.RUnlock()
+
+ for k, v := range header {
+ req.Header[k] = v
+ }
+
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ return &Response{resp}, nil
+}
+
+
+
package accounting
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+)
+
+type GetSyncQuotaReq struct {
+ AccessToken string `json:"access_token"`
+}
+
+type SyncQuota struct {
+ RepoCountLimit int64 `json:"repo_count_limit"`
+ TrafficLimit int64 `json:"traffic_limit"`
+ AccessToken string `json:"-"`
+ RepoCountUsed int64 `json:"repo_count_used"`
+ SpeedLimit int64 `json:"speed_limit"`
+ TrafficUsed int64 `json:"traffic_used"`
+}
+
+type SyncQuotaRes struct {
+ Message string `json:"msg"`
+ Data SyncQuota `json:"data"`
+}
+
+type CreateSyncQuotaReq = SyncQuota
+
+type UpdateSyncQuotaReq = SyncQuota
+
+func (c *AccountingClient) CreateOrUpdateSyncQuota(opt *CreateSyncQuotaReq) (*Response, error) {
+ header := http.Header{"content-type": []string{"application/json"}}
+ body, err := json.Marshal(&opt)
+ if err != nil {
+ return nil, err
+ }
+ if opt.AccessToken != "" {
+ header.Add("Authorization", "Bearer "+opt.AccessToken)
+ }
+ _, resp, err := c.getResponse("POST", "/accounting/multisync/quotas", header, bytes.NewReader(body))
+ return resp, err
+}
+
+func (c *AccountingClient) GetSyncQuota(opt *GetSyncQuotaReq) (*SyncQuota, *Response, error) {
+ s := new(SyncQuotaRes)
+ header := http.Header{}
+ if opt.AccessToken != "" {
+ header.Add("Authorization", "Bearer "+opt.AccessToken)
+ }
+ resp, err := c.getParsedResponse("GET", "/accounting/multisync/quota", header, nil, s)
+ return &s.Data, resp, err
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/multisync/accounting"
+ "opencsg.com/csghub-server/multisync/types"
+)
+
+type MirrorProxyComponent struct {
+ ac *accounting.AccountingClient
+ user database.UserStore
+}
+
+func NewMirrorProxyComponent(config *config.Config) (*MirrorProxyComponent, error) {
+ ac, err := accounting.NewAccountingClient(config)
+ if err != nil {
+ return nil, err
+ }
+ return &MirrorProxyComponent{
+ ac: ac,
+ user: database.NewUserStore(),
+ }, nil
+}
+
+func (c *MirrorProxyComponent) Serve(ctx context.Context, req *types.GetSyncQuotaStatementReq) error {
+ sq, _, err := c.ac.GetSyncQuota(&accounting.GetSyncQuotaReq{
+ AccessToken: req.AccessToken,
+ })
+ if err != nil {
+ return fmt.Errorf("error getting sync quota: %v", err)
+ }
+ if sq.RepoCountLimit <= sq.RepoCountUsed {
+ return fmt.Errorf("sync repository count limit exceeded")
+ }
+ sqs, _, err := c.ac.GetSyncQuotaStatement(&accounting.GetSyncQuotaStatementsReq{
+ AccessToken: req.AccessToken,
+ RepoPath: req.RepoPath,
+ RepoType: req.RepoType,
+ })
+ if err != nil {
+ return fmt.Errorf("error getting sync quota statement: %v", err)
+ }
+ if sqs.ID != 0 {
+ return nil
+ }
+ resp, err := c.ac.CreateSyncQuotaStatement(&accounting.CreateSyncQuotaStatementReq{
+ AccessToken: req.AccessToken,
+ RepoPath: req.RepoPath,
+ RepoType: req.RepoType,
+ })
+ if err != nil {
+ return fmt.Errorf("error creating sync quota statement: %v", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return fmt.Errorf("error creating sync quota statement")
+ }
+ return nil
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/proxy"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/multisync/component"
+ "opencsg.com/csghub-server/multisync/types"
+)
+
+const MirrorTokenHeaderKey = "X-OPENCSG-Sync-Token"
+
+type MirrorProxyHandler struct {
+ gitServerURL string
+ mpComp *component.MirrorProxyComponent
+}
+
+func NewMirrorProxyHandler(config *config.Config) (*MirrorProxyHandler, error) {
+ mpComp, err := component.NewMirrorProxyComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create repo component,%w", err)
+ }
+
+ return &MirrorProxyHandler{
+ mpComp: mpComp,
+ gitServerURL: config.GitServer.URL,
+ }, nil
+}
+
+func (r *MirrorProxyHandler) Serve(ctx *gin.Context) {
+ var req types.GetSyncQuotaStatementReq
+ token := getMirrorTokenFromContext(ctx)
+ repoType := ctx.Param("repo_type")
+ namespace := ctx.Param("namespace")
+ name := ctx.Param("name")
+ name, _ = strings.CutSuffix(name, ".git")
+ req.RepoPath = fmt.Sprintf("%s/%s", namespace, name)
+ req.RepoType = strings.TrimSuffix(repoType, "s")
+ req.AccessToken = token
+
+ if strings.HasSuffix(ctx.Request.URL.Path, "git-upload-pack") {
+ err := r.mpComp.Serve(ctx, &req)
+ if err != nil {
+ slog.Error("failed to serve git upload pack request:", slog.Any("err", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ }
+
+ rp, _ := proxy.NewReverseProxy(r.gitServerURL)
+ rp.ServeHTTP(ctx.Writer, ctx.Request, ctx.Request.URL.Path)
+}
+
+func (r *MirrorProxyHandler) ServeLFS(ctx *gin.Context) {
+ rp, _ := proxy.NewReverseProxy(r.gitServerURL)
+ rp.ServeHTTP(ctx.Writer, ctx.Request, ctx.Request.URL.Path)
+}
+
+func getMirrorTokenFromContext(ctx *gin.Context) string {
+ csgHeaderToken := ctx.GetHeader(MirrorTokenHeaderKey)
+ if csgHeaderToken != "" {
+ return csgHeaderToken
+ }
+ authToken := ctx.GetHeader("Authorization")
+ authToken = strings.TrimPrefix(authToken, "Bearer ")
+ authToken = strings.TrimPrefix(authToken, MirrorTokenHeaderKey)
+ return authToken
+}
+
+
+
package router
+
+import (
+ "fmt"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/multisync/handler"
+)
+
+func NewRouter(config *config.Config) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+ // store := cookie.NewStore([]byte(config.Mirror.SessionSecretKey))
+ // store.Options(sessions.Options{
+ // SameSite: http.SameSiteNoneMode,
+ // Secure: config.EnableHTTPS,
+ // })
+ // r.Use(sessions.Sessions("jwt_session", store))
+ // r.Use(middleware.BuildJwtSession(config.JWT.SigningKey))
+
+ mpHandler, err := handler.NewMirrorProxyHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating rproxy handler:%w", err)
+ }
+ rGroup := r.Group("/:repo_type/:namespace/:name")
+ {
+ rGroup.POST("/git-upload-pack", mpHandler.Serve)
+ rGroup.POST("/git-receive-pack", mpHandler.Serve)
+ rGroup.GET("/info/refs", mpHandler.Serve)
+ rGroup.GET("/HEAD", mpHandler.Serve)
+ rGroup.GET("/objects/info/alternates", mpHandler.Serve)
+ rGroup.GET("/objects/info/http-alternates", mpHandler.Serve)
+ rGroup.GET("/objects/info/packs", mpHandler.Serve)
+ rGroup.GET("/objects/info/:file", mpHandler.Serve)
+ rGroup.GET("/objects/:head/:hash", mpHandler.Serve)
+ rGroup.GET("/objects/pack/pack-:file", mpHandler.Serve)
+ rGroup.POST("/info/lfs/objects/batch", mpHandler.ServeLFS)
+ rGroup.GET("/info/lfs/objects/:oid", mpHandler.ServeLFS)
+ }
+
+ // r.Any("/*api", handler.Serve)
+
+ return r, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "log/slog"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+ "time"
+)
+
+func updatePaymentAndNotify(
+ c context.Context,
+ store *database.PaymentStore,
+ eventPub *event.EventPublisher,
+ outTradeNo string,
+ tradeNo string,
+ checkFn func(payment *database.Payment) bool,
+ updateFn func(payment *database.Payment),
+) error {
+ payment, err := store.GetPaymentByOrderNo(c, outTradeNo)
+ if err != nil {
+ slog.Error(err.Error())
+ return err
+ }
+
+ if !checkFn(payment) {
+ return nil
+ }
+
+ updateFn(payment)
+
+ err = store.UpdatePayment(c, payment)
+ if err != nil {
+ slog.Error(err.Error())
+ return err
+ }
+
+ notifyEvent := types.PaymentNotifyEvent{
+ PaymentUUID: payment.PaymentUUID,
+ TransactionNo: payment.TransactionNo,
+ OrderNo: payment.OrderNo,
+ Channel: string(payment.Channel),
+ Paid: payment.Paid,
+ Reversed: payment.Reversed,
+ TimePaid: payment.TimePaid,
+ TimeExpire: payment.TimeExpire,
+ Amount: payment.Amount,
+ Extra: payment.Extra,
+ }
+
+ eventData, err := json.Marshal(notifyEvent)
+ if err != nil {
+ slog.Error("Error marshaling payment notification event", slog.Any("event", notifyEvent), slog.Any("error", err))
+ return err
+ }
+
+ err = eventPub.PublishRechargeEvent(eventData)
+ if err != nil {
+ slog.Error("Failed to publish payment notification event", slog.Any("data", string(eventData)), slog.Any("error", err))
+ return err
+ }
+
+ slog.Debug("Payment notification event published successfully", slog.Any("data", string(eventData)))
+ return nil
+}
+
+func ProcessPaymentSuccess(c context.Context, store *database.PaymentStore, eventPub *event.EventPublisher, outTradeNo string, tradeNo string) error {
+ return updatePaymentAndNotify(
+ c,
+ store,
+ eventPub,
+ outTradeNo,
+ tradeNo,
+ func(payment *database.Payment) bool {
+ if payment.Paid {
+ slog.Info("Payment already processed")
+ return false
+ }
+ return true
+ },
+ func(payment *database.Payment) {
+ payment.Paid = true
+ payment.TimePaid = time.Now()
+ payment.UpdatedAt = time.Now()
+ payment.TransactionNo = tradeNo
+ },
+ )
+}
+
+func ProcessPaymentClose(c context.Context, store *database.PaymentStore, eventPub *event.EventPublisher, outTradeNo string, tradeNo string) error {
+ return updatePaymentAndNotify(
+ c,
+ store,
+ eventPub,
+ outTradeNo,
+ tradeNo,
+ func(payment *database.Payment) bool {
+ if payment.Paid {
+ slog.Error("Payment already paid! but fetch fail state")
+ return false
+ }
+ if payment.Reversed {
+ slog.Info("Payment already reversed")
+ return false
+ }
+ return true
+ },
+ func(payment *database.Payment) {
+ payment.Reversed = true
+ payment.UpdatedAt = time.Now()
+ payment.TimeExpire = time.Now()
+ payment.TransactionNo = tradeNo
+ },
+ )
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "github.com/google/uuid"
+ "log/slog"
+ csghubevent "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ impl "opencsg.com/csghub-server/payment/gateway"
+ "opencsg.com/csghub-server/payment/gatewayfactory"
+ "time"
+)
+
+type PaymentComponent struct {
+ ps *database.PaymentStore
+ config *config.Config
+}
+
+func NewPaymentComponent(config *config.Config) *PaymentComponent {
+ pc := &PaymentComponent{
+ ps: database.NewPaymentStore(),
+ config: config,
+ }
+ return pc
+}
+
+func (pc *PaymentComponent) CreateSimplePayment(
+ ctx context.Context,
+ orderNo string,
+ subject string,
+ body string,
+ amount *money.Money,
+ extra string,
+ channel consts.PaymentChannel) (*database.Payment, error) {
+
+ payment := &database.Payment{}
+ payment.PaymentUUID = uuid.NewString()
+ payment.OrderNo = orderNo
+ payment.Amount = amount.GetAmount()
+ payment.Channel = channel
+ payment.Subject = subject
+ payment.Body = body
+ payment.Extra = extra
+
+ gateway, err := gatewayfactory.GetPaymentGateway(channel)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get payment gateway %s, error: %w", channel, err)
+ }
+ paymentExpireIn := pc.config.Payment.PaymentExpireIn
+ expiredIn := time.Duration(paymentExpireIn) * time.Second
+
+ resp, err := gateway.GenerateQRCode(ctx, payment.OrderNo, amount, payment.Subject, expiredIn)
+ if err != nil {
+ return nil, fmt.Errorf("failed to add price, error: %w", err)
+ }
+ payment.CodeUrl = resp.QRCodeURL
+ err = pc.ps.CreatePayment(ctx, payment)
+ if err != nil {
+ return nil, fmt.Errorf("failed to add price, error: %w", err)
+ }
+ // Start monitoring the payment without blocking
+ after := 5 * time.Second
+ buffer := 60 * time.Second //add 1min buffer time
+ timeOut := expiredIn + buffer
+ impl.StartPaymentMonitor(ctx, gateway, payment.OrderNo, after, timeOut,
+ func(result impl.PaymentResult) {
+ if result.Status == consts.PaymentSuccess {
+ e := ProcessPaymentSuccess(ctx,
+ database.NewPaymentStore(),
+ &csghubevent.DefaultEventPublisher,
+ payment.OrderNo,
+ result.TradeNo,
+ )
+ if e != nil {
+ slog.Error(err.Error())
+ }
+ } else if result.Status == consts.PaymentClosed {
+ e := ProcessPaymentClose(ctx,
+ database.NewPaymentStore(),
+ &csghubevent.DefaultEventPublisher,
+ payment.OrderNo,
+ result.TradeNo,
+ )
+ if e != nil {
+ slog.Error(err.Error())
+ }
+ }
+ })
+ return payment, nil
+}
+
+
+
package alipay
+
+import (
+ "context"
+ "fmt"
+ "github.com/go-pay/gopay"
+ "log/slog"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "opencsg.com/csghub-server/payment/gateway"
+ "time"
+)
+
+type Gateway struct {
+ clientManager *AliPayConfig
+}
+
+func NewGateway() (*Gateway, error) {
+ clientManager, err := AliClientV3Manager()
+ if err != nil {
+ slog.Error("fail to create alipay client", slog.Any("err", err))
+ return nil, err
+ }
+ return &Gateway{clientManager: clientManager}, nil
+}
+
+func (g *Gateway) generateTimeExpire(duration time.Duration) (string, error) {
+ beijingLocation, err := time.LoadLocation("Asia/Shanghai")
+ if err != nil {
+ return "", err
+ }
+ expireTime := time.Now().In(beijingLocation).Add(duration)
+ timeExpire := expireTime.Format("2006-01-02 15:04:05")
+ return timeExpire, nil
+}
+
+// GenerateQRCode https://github.com/go-pay/gopay/blob/main/doc/alipay_v3.md
+// https://opendocs.alipay.com/open/05osuz?pathHash=fa169dd3
+func (g *Gateway) GenerateQRCode(ctx context.Context, outTradeNo string, amount *money.Money, description string, expireIn time.Duration) (gateway.QRCodePayment, error) {
+ client, err := g.clientManager.GetClient()
+ if err != nil {
+ slog.Error("get alipay client fail", slog.Any("err", err))
+ return gateway.QRCodePayment{}, err
+ }
+
+ amountYuan, err := amount.ToYuanString()
+ if err != nil {
+ slog.Error("convert amount to yuan failed", slog.Any("err", err))
+ return gateway.QRCodePayment{}, err
+ }
+
+ timeExpire, err := g.generateTimeExpire(expireIn)
+ if err != nil {
+ slog.Error("alipay trade pre create generate time_expire failed", slog.Any("err", err))
+ return gateway.QRCodePayment{}, err
+ }
+
+ bm := make(gopay.BodyMap)
+ bm.Set("subject", description).
+ Set("out_trade_no", outTradeNo).
+ Set("total_amount", amountYuan).
+ Set("time_expire", timeExpire).
+ Set("notify_url", g.clientManager.AlipayNotifyUrl)
+
+ rsp, err := client.TradePrecreate(ctx, bm)
+ if err != nil {
+ slog.Error("alipay trade pre create fail", slog.Any("err", err))
+ return gateway.QRCodePayment{}, err
+ }
+
+ if rsp.StatusCode == 200 {
+ codeUrl := rsp.QrCode
+ p := gateway.QRCodePayment{QRCodeURL: codeUrl, OutTradeNo: outTradeNo}
+ return p, nil
+ } else {
+ errResp := rsp.ErrResponse
+ return gateway.QRCodePayment{}, fmt.Errorf("alipay error: %v", errResp)
+ }
+}
+
+func (g *Gateway) WaitForPayment(ctx context.Context, outTradeNo string, after time.Duration, expiredIn time.Duration) gateway.PaymentResult {
+ time.Sleep(after)
+ client, err := g.clientManager.GetClient()
+ if err != nil {
+ slog.Error("fail to get alipay client", slog.Any("error", err))
+ return gateway.PaymentResult{Status: consts.PaymentFailed, Err: err}
+ }
+
+ startTime := time.Now()
+ for {
+ // Check if context is done
+ if ctx.Err() != nil {
+ return gateway.PaymentResult{Status: consts.PaymentFailed, Err: ctx.Err()}
+ }
+
+ bm := make(gopay.BodyMap)
+ bm.Set("out_trade_no", outTradeNo)
+
+ rsp, err := client.TradeQuery(ctx, bm)
+ if err != nil {
+ slog.Error("fail to invoke alipay trade query api", slog.Any("error", err))
+ return gateway.PaymentResult{Status: consts.PaymentFailed, Err: err}
+ }
+
+ if rsp.StatusCode == 200 {
+ tradeStatus := consts.TradeStatus(rsp.TradeStatus)
+ tradeNo := rsp.TradeNo
+ switch tradeStatus {
+ case consts.TradeStatusTradeSuccess, consts.TradeStatusTradeFinished:
+ return gateway.PaymentResult{Status: consts.PaymentSuccess, TradeNo: tradeNo}
+ case consts.TradeStatusWaitBuyerPay:
+ if time.Since(startTime) >= expiredIn {
+ return gateway.PaymentResult{Status: consts.PaymentClosed, TradeNo: tradeNo}
+ }
+ time.Sleep(5 * time.Second)
+ case consts.TradeStatusTradeClosed:
+ return gateway.PaymentResult{Status: consts.PaymentClosed, TradeNo: tradeNo}
+ default:
+ e := fmt.Errorf("unknown trade status: %s", tradeStatus)
+ return gateway.PaymentResult{Status: consts.PaymentFailed, TradeNo: tradeNo, Err: e}
+ }
+ } else {
+ if rsp.StatusCode == 400 && rsp.ErrResponse.Code == "ACQ.TRADE_NOT_EXIST" {
+ if time.Since(startTime) >= expiredIn {
+ return gateway.PaymentResult{Status: consts.PaymentClosed}
+ }
+ time.Sleep(5 * time.Second)
+ } else {
+ e := fmt.Errorf("alipay fetch outTradeOrder %s error: %v", outTradeNo, rsp)
+ return gateway.PaymentResult{Status: consts.PaymentFailed, Err: e}
+ }
+ }
+ }
+}
+
+func (g *Gateway) ClosePayment(ctx context.Context, outTradeNo string) error {
+ slog.Debug("Closing Alipay order, outTradeNo: " + outTradeNo)
+ client, err := g.clientManager.GetClient()
+ if err != nil {
+ slog.Error("fail to get client", slog.Any("error", err))
+ return err
+ }
+
+ bm := make(gopay.BodyMap)
+ bm.Set("out_trade_no", outTradeNo)
+
+ rsp, err := client.TradeClose(ctx, bm)
+ if err != nil {
+ slog.Error("fail to invoke alipay trade close api", slog.Any("error", err))
+ return err
+ }
+
+ if rsp.StatusCode == 200 {
+ return nil
+ } else {
+ return fmt.Errorf("alipay error: %v", rsp)
+ }
+}
+
+
+
package alipay
+
+import (
+ "fmt"
+ "github.com/go-pay/gopay"
+ "github.com/go-pay/gopay/alipay/v3"
+ "log/slog"
+ "opencsg.com/csghub-server/common/config"
+)
+
+// AliPayConfig holds configuration for AliPay client
+type AliPayConfig struct {
+ IsProd bool
+ AppId string
+ AppPrivateKey string
+ AlipayPublicKey string
+
+ AppPublicCert []byte
+ AlipayRootCert []byte
+ AlipayPublicCert []byte
+
+ AppPublicCertPath string
+ AlipayRootCertPath string
+ AlipayPublicCertPath string
+
+ AlipayNotifyUrl string
+}
+
+func AliClientV3Manager() (*AliPayConfig, error) {
+ conf, err := config.LoadConfig()
+ if err != nil {
+ return nil, err
+ }
+ domain := conf.APIServer.PublicDomain
+ path := conf.Payment.AlipayNotifyPath
+ notifyUrl := fmt.Sprintf("%s%s", domain, path)
+ clientConfig := &AliPayConfig{
+ AppId: conf.Payment.AlipayAppId,
+ AppPrivateKey: conf.Payment.AlipayPrivateKey,
+ AlipayPublicKey: conf.Payment.AlipayPublicKey,
+ IsProd: conf.Payment.IsProd,
+ AlipayNotifyUrl: notifyUrl,
+ }
+ return clientConfig, nil
+}
+
+// GetClient returns a configured WeChat Pay client instance
+func (w *AliPayConfig) GetClient() (*alipay.ClientV3, error) {
+ client, err := alipay.NewClientV3(w.AppId, w.AppPrivateKey, true)
+ if err != nil {
+ slog.Error(err.Error())
+ return nil, err
+ }
+ if w.IsProd {
+ client.DebugSwitch = gopay.DebugOff
+ } else {
+ client.DebugSwitch = gopay.DebugOn
+ }
+ // Enable automatic response signature verification
+ //cert.AppPublicContent
+ //err = client.SetCert(w.AppPublicCert, w.AlipayRootCert, w.AlipayPublicCert)
+
+ return client, err
+}
+
+
+
package gateway
+
+import "time"
+import "context"
+import "log/slog"
+
+func StartPaymentMonitor(ctx context.Context, gateway PaymentGateway, outTradeNo string, after time.Duration, expiredIn time.Duration, callback func(PaymentResult)) {
+ go func() {
+ slog.Info("Payment status monitor begin", slog.String("payment", outTradeNo))
+ result := gateway.WaitForPayment(ctx, outTradeNo, after, expiredIn)
+ slog.Info("Payment status monitor succeed", slog.Any("payment", outTradeNo), slog.Any("result", result))
+ callback(result)
+ slog.Info("Payment status monitor finish", slog.String("payment", outTradeNo))
+ }()
+}
+
+
+
package wechat
+
+import (
+ "context"
+ "fmt"
+ "github.com/go-pay/gopay"
+ "github.com/go-pay/gopay/wechat/v3"
+ "log/slog"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "opencsg.com/csghub-server/payment/gateway"
+ "time"
+)
+
+type Gateway struct {
+ clientManager *WxPayConfig
+}
+
+func NewGateway() (*Gateway, error) {
+ clientManager, err := WxClientV3Manager()
+ if err != nil {
+ slog.Error("fail to create wechat pay client manager", slog.Any("err", err))
+ return nil, err
+ }
+ return &Gateway{clientManager: clientManager}, nil
+}
+
+func (g *Gateway) generateTimeExpire(expireIn time.Duration) string {
+ expireTime := time.Now().Add(expireIn)
+ return expireTime.Format(time.RFC3339)
+}
+
+// https://pay.weixin.qq.com/docs/merchant/products/native-payment/development.html
+func (g *Gateway) GenerateQRCode(ctx context.Context, outTradeNo string, amount *money.Money, description string, expireIn time.Duration) (gateway.QRCodePayment, error) {
+ slog.Debug("tradeNo:" + outTradeNo)
+
+ expire := g.generateTimeExpire(expireIn)
+
+ client, err := g.clientManager.GetClient()
+ if err != nil {
+ slog.Error("fail to create wechat pay client", slog.Any("err", err))
+ return gateway.QRCodePayment{}, err
+ }
+
+ bm := make(gopay.BodyMap)
+ bm.Set("appid", g.clientManager.AppId).
+ Set("description", description).
+ Set("out_trade_no", outTradeNo).
+ Set("time_expire", expire).
+ Set("notify_url", g.clientManager.WxPayNotifyUrl).
+ SetBodyMap("amount", func(bm gopay.BodyMap) {
+ bm.Set("total", amount.GetAmount()).Set("currency", amount.GetCurrency())
+ })
+
+ wxRsp, e := client.V3TransactionNative(ctx, bm)
+ if e != nil {
+ slog.Error("fail to create wechat v3 transaction native", slog.Any("err", e))
+ return gateway.QRCodePayment{}, e
+ }
+
+ if wxRsp.Code == wechat.Success {
+ codeUrl := wxRsp.Response.CodeUrl
+ p := gateway.QRCodePayment{QRCodeURL: codeUrl, OutTradeNo: outTradeNo}
+ return p, nil
+ } else {
+ slog.Error("fail to create wechat v3 transaction native", slog.Any("wxRsp err", wxRsp.Error))
+ return gateway.QRCodePayment{}, fmt.Errorf("wechat pay error: %v", wxRsp.Error)
+ }
+}
+
+// https://pay.weixin.qq.com/docs/merchant/apis/native-payment/close-order.html
+
+func (g *Gateway) WaitForPayment(ctx context.Context, outTradeNo string, after time.Duration, expiredIn time.Duration) gateway.PaymentResult {
+ time.Sleep(after)
+ client, err := g.clientManager.GetClient()
+ if err != nil {
+ slog.Error("fail to get wechat v3 pay client manager", slog.Any("err", err))
+ return gateway.PaymentResult{Err: err, Status: consts.PaymentFailed}
+ }
+
+ startTime := time.Now()
+ for {
+ // Check if context is done
+ if ctx.Err() != nil {
+ return gateway.PaymentResult{Err: ctx.Err(), Status: consts.PaymentFailed}
+ }
+
+ wxRsp, err := client.V3TransactionQueryOrder(ctx, wechat.OutTradeNo, outTradeNo)
+ if err != nil {
+ slog.Error("fail to query wechat v3 transaction order", slog.Any("err", err))
+ return gateway.PaymentResult{Err: err, Status: consts.PaymentFailed}
+ }
+
+ if wxRsp.Code == wechat.Success {
+ tradeState := wxRsp.Response.TradeState
+ tradeNo := wxRsp.Response.TransactionId
+ switch tradeState {
+ case consts.TradeStatusSuccess:
+ return gateway.PaymentResult{Status: consts.PaymentSuccess, TradeNo: tradeNo}
+ case consts.TradeStatusNotPay, consts.TradeStatusUserPaying:
+ if time.Since(startTime) >= expiredIn {
+ return gateway.PaymentResult{Status: consts.PaymentClosed, TradeNo: tradeNo}
+ }
+ time.Sleep(5 * time.Second)
+ case consts.TradeStatusClosed, consts.TradeStatusRevoked, consts.TradeStatusPayError:
+ return gateway.PaymentResult{Status: consts.PaymentClosed, TradeNo: tradeNo}
+ default:
+ return gateway.PaymentResult{Err: fmt.Errorf("unknown trade state: %s", tradeState), Status: consts.PaymentFailed, TradeNo: tradeNo}
+ }
+ } else {
+ slog.Error("fail to query wechat v3 transaction order", slog.Any("wxRsp err", wxRsp.Error))
+ return gateway.PaymentResult{Err: fmt.Errorf("wechat payment fetch error: %v", wxRsp), Status: consts.PaymentFailed}
+ }
+ }
+}
+
+func (g *Gateway) ClosePayment(ctx context.Context, outTradeNo string) error {
+ slog.Debug("Closing WeChat Pay order, tradeNo: " + outTradeNo)
+ client, err := g.clientManager.GetClient()
+ if err != nil {
+ slog.Error("fail to create wechat pay client", slog.Any("err", err))
+ return err
+ }
+
+ rsp, err := client.V3TransactionCloseOrder(ctx, outTradeNo)
+ if err != nil {
+ slog.Error("fail to close wechat v3 transaction order", slog.Any("err", err))
+ return err
+ }
+
+ if rsp.Code == wechat.Success {
+ return nil
+ } else {
+ slog.Error("fail to close wechat v3 transaction order", slog.Any("rsp", rsp))
+ return fmt.Errorf("wechat pay error: %v", rsp)
+ }
+}
+
+
+
package wechat
+
+import (
+ "fmt"
+ "github.com/go-pay/gopay"
+ "github.com/go-pay/gopay/wechat/v3"
+ "log/slog"
+ "opencsg.com/csghub-server/common/config"
+ "os"
+)
+
+// WxPayConfig holds configuration for WeChat Pay client
+type WxPayConfig struct {
+ AppId string
+ MchId string
+ MchCertificateSerialNumber string
+ MchAPIv3Key string
+ PrivateKeyContentStr string
+ WxPayNotifyUrl string
+ IsProd bool
+}
+
+// Option defines a function type for setting WxPayConfig options
+type Option func(*WxPayConfig)
+
+// WithAppId sets the AppId
+func WithAppId(appId string) Option {
+ return func(w *WxPayConfig) {
+ w.AppId = appId
+ }
+}
+
+// WithMchId sets the MchId
+func WithMchId(mchId string) Option {
+ return func(w *WxPayConfig) {
+ w.MchId = mchId
+ }
+}
+
+// WithMchCertificateSerialNumber sets the certificate serial number
+func WithMchCertificateSerialNumber(serial string) Option {
+ return func(w *WxPayConfig) {
+ w.MchCertificateSerialNumber = serial
+ }
+}
+
+// WithMchAPIv3Key sets the API v3 key
+func WithMchAPIv3Key(apiV3Key string) Option {
+ return func(w *WxPayConfig) {
+ w.MchAPIv3Key = apiV3Key
+ }
+}
+
+// WithPrivateKeyContentStr sets the private key content
+func WithPrivateKeyContentStr(privateKeyPath string) Option {
+ return func(w *WxPayConfig) {
+ content, err := os.ReadFile(privateKeyPath)
+ if err == nil {
+ w.PrivateKeyContentStr = string(content)
+ }
+ }
+}
+
+// WithWxPayNotifyUrl sets the notification URL
+func WithWxPayNotifyUrl(notifyUrl string) Option {
+ return func(w *WxPayConfig) {
+ w.WxPayNotifyUrl = notifyUrl
+ }
+}
+
+// WxClientV3Manager initializes WxPayConfig with functional options
+func WxClientV3Manager(opts ...Option) (*WxPayConfig, error) {
+ conf, _ := config.LoadConfig()
+ domain := conf.APIServer.PublicDomain
+ path := conf.Payment.WXPayNotifyPath
+ notifyUrl := fmt.Sprintf("%s%s", domain, path)
+ clientConfig := &WxPayConfig{
+ AppId: conf.Payment.WXAppId,
+ MchId: conf.Payment.WXMchId,
+ MchCertificateSerialNumber: conf.Payment.WXMchCertificateSerialNumber,
+ MchAPIv3Key: conf.Payment.WXMchAPIv3Key,
+ IsProd: conf.Payment.IsProd,
+ WxPayNotifyUrl: notifyUrl,
+ }
+
+ // Apply provided options
+ for _, opt := range opts {
+ opt(clientConfig)
+ }
+
+ // If PrivateKeyContentStr is not set by options, load from default path
+ if clientConfig.PrivateKeyContentStr == "" {
+ content, err := os.ReadFile(conf.Payment.WXMchCertificatePrivateKeyFilePath)
+ if err != nil {
+ return nil, err
+ }
+ clientConfig.PrivateKeyContentStr = string(content)
+ }
+ return clientConfig, nil
+}
+
+// GetClient returns a configured WeChat Pay client instance
+func (w *WxPayConfig) GetClient() (*wechat.ClientV3, error) {
+ client, err := wechat.NewClientV3(w.MchId, w.MchCertificateSerialNumber, w.MchAPIv3Key, w.PrivateKeyContentStr)
+ if err != nil {
+ slog.Error("get v3 client error", slog.Any("err", err))
+ return nil, err
+ }
+
+ // Enable automatic response signature verification
+ err = client.AutoVerifySign()
+ if err != nil {
+ slog.Error("AutoVerifySign error", slog.Any("err", err))
+ return nil, err
+ }
+
+ if w.IsProd {
+ client.DebugSwitch = gopay.DebugOff
+ } else {
+ client.DebugSwitch = gopay.DebugOn
+ }
+
+ return client, nil
+}
+
+
+
package gatewayfactory
+
+import (
+ "fmt"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "opencsg.com/csghub-server/payment/gateway"
+ "opencsg.com/csghub-server/payment/gateway/alipay"
+ "opencsg.com/csghub-server/payment/gateway/wechat"
+)
+
+func GetPaymentGateway(paymentChannel consts.PaymentChannel) (gateway.PaymentGateway, error) {
+ switch paymentChannel {
+ case consts.ChannelAlipayQr:
+ return alipay.NewGateway()
+ case consts.ChannelWxPubQr:
+ return wechat.NewGateway()
+ default:
+ return nil, fmt.Errorf("unsupported payment channel: %s", paymentChannel)
+ }
+}
+
+
+
package handler
+
+import (
+ "github.com/gin-gonic/gin"
+ "github.com/go-pay/gopay/alipay"
+ "log/slog"
+ "net/http"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/payment/component"
+ alipayClient "opencsg.com/csghub-server/payment/gateway/alipay"
+)
+
+type AliPayNotifyHandler struct {
+ eventPub *event.EventPublisher
+}
+
+// https://opendocs.alipay.com/open/0c2c19?pathHash=df12a335
+// https://opendocs.alipay.com/open/203/105286
+// https://github.com/go-pay/gopay/blob/main/doc/alipay_v3.md
+func (h *AliPayNotifyHandler) AliPayNotify(c *gin.Context) {
+ bm, err := alipay.ParseNotifyToBodyMap(c.Request)
+ if err != nil {
+ slog.Error("Alipay parse notify content failed", slog.Any("err", err))
+ c.String(http.StatusBadRequest, "error")
+ return
+ }
+
+ manager, err := alipayClient.AliClientV3Manager()
+ if err != nil {
+ slog.Error("Failed to get AliClientV3Manager", slog.Any("err", err))
+ c.String(http.StatusInternalServerError, "error")
+ return
+ }
+
+ ok, err := alipay.VerifySign(manager.AlipayPublicKey, bm)
+ if err != nil {
+ slog.Error("Alipay signature verification failed", slog.Any("err", err))
+ c.String(http.StatusInternalServerError, "error")
+ return
+ }
+
+ if !ok {
+ slog.Error("Alipay notify verify sign failed: signature mismatch")
+ c.String(http.StatusOK, "fail")
+ return
+ }
+
+ tradeStatus := bm.GetString("trade_status")
+ outTradeNo := bm.GetString("out_trade_no")
+ tradeNo := bm.GetString("trade_no")
+
+ if tradeStatus == "TRADE_SUCCESS" {
+ slog.Info("Processing successful Alipay payment",
+ slog.String("outTradeNo", outTradeNo),
+ slog.String("tradeNo", tradeNo))
+
+ err := component.ProcessPaymentSuccess(c, database.NewPaymentStore(), h.eventPub, outTradeNo, tradeNo)
+ if err != nil {
+ slog.Error("Process Alipay payment success failed", slog.Any("err", err))
+ c.String(http.StatusOK, "fail")
+ return
+ }
+ c.String(http.StatusOK, "success")
+ } else {
+ slog.Info("Alipay payment status not successful", slog.String("tradeStatus", tradeStatus))
+ c.String(http.StatusOK, "fail")
+ }
+}
+
+func NewAliPayNotifyHandler(config *config.Config) (*AliPayNotifyHandler, error) {
+ return &AliPayNotifyHandler{
+ eventPub: &event.DefaultEventPublisher,
+ }, nil
+}
+
+
+
package handler
+
+import (
+ "github.com/gin-gonic/gin"
+ "log/slog"
+ "net/http"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/money"
+ "opencsg.com/csghub-server/payment/component"
+)
+
+type PaymentHandler struct {
+ paymentComp *component.PaymentComponent
+}
+
+func NewPaymentHandler(config *config.Config) (*PaymentHandler, error) {
+ handler := PaymentHandler{
+ paymentComp: component.NewPaymentComponent(config),
+ }
+ return &handler, nil
+}
+
+// POST
+// CreatePayment godoc
+// @Security ApiKey
+// @Summary Create payment order
+// @Description Create payment order
+// @Tags Payment
+// @Accept json
+// @Produce json
+// @Param body body types.CreatePaymentReq{} true
+// @Success 200 {object} types.CreatePaymentResp{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /payment/simple-payments [post]
+func (h *PaymentHandler) CreateSimplePayment(ctx *gin.Context) {
+
+ var req types.CreatePaymentReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("Bad payment request format", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ amount, err := money.NewMoneyFromYuan(req.Amount)
+
+ if err != nil {
+ slog.Error("Bad recharge request format for recharge amount", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ pc := h.paymentComp
+ payment, err := pc.CreateSimplePayment(ctx, req.OrderNo, req.Subject, req.Body, amount, req.Extra, req.Channel)
+ if err != nil {
+ slog.Error("create payment fail", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ rsp := types.CreatePaymentResp{
+ PaymentUUID: payment.PaymentUUID,
+ OrderNo: payment.OrderNo,
+ CodeUrl: payment.CodeUrl,
+ }
+ ctx.PureJSON(http.StatusOK, rsp)
+}
+
+
+
package handler
+
+import (
+ "github.com/gin-gonic/gin"
+ "github.com/go-pay/gopay"
+ "github.com/go-pay/gopay/wechat/v3"
+ "log/slog"
+ "net/http"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/utils/payment/consts"
+ "opencsg.com/csghub-server/payment/component"
+ wechatGateway "opencsg.com/csghub-server/payment/gateway/wechat"
+)
+
+type WechatPayNotifyHandler struct {
+ eventPub *event.EventPublisher
+}
+
+// https://pay.weixin.qq.com/docs/merchant/apis/native-payment/payment-notice.html
+// https://pay.weixin.qq.com/docs/merchant/development/practices/callback-and-query-order.html
+func (h *WechatPayNotifyHandler) WechatPayNotify(c *gin.Context) {
+ notifyReq, err := wechat.V3ParseNotify(c.Request)
+
+ if err != nil {
+ slog.Error("wechat parse notify content fail", slog.Any("err", err))
+ c.JSON(http.StatusBadRequest, gin.H{"code": gopay.FAIL, "message": "Failed to parse notification"})
+ return
+ }
+
+ manager, err := wechatGateway.WxClientV3Manager()
+ if err != nil {
+ slog.Error("failed to get WxClientV3Manager", slog.Any("err", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"code": gopay.FAIL, "message": "Internal server error"})
+ return
+ }
+ client, err := manager.GetClient()
+ if err != nil {
+ slog.Error("failed to get WxClientV3 client", slog.Any("err", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"code": gopay.FAIL, "message": "Internal server error"})
+ return
+ }
+ certMap := client.WxPublicKeyMap()
+
+ err = notifyReq.VerifySignByPKMap(certMap)
+ if err != nil {
+ slog.Error("wechat VerifySignByPKMap fail", slog.Any("err", err))
+ c.JSON(http.StatusBadRequest, gin.H{"code": gopay.FAIL, "message": "Failed to decrypt notification data"})
+ return
+ }
+
+ result, _ := notifyReq.DecryptPayCipherText(manager.MchAPIv3Key)
+ outTradeNo := result.OutTradeNo
+ tradeNo := result.TransactionId
+ tradeType := result.TradeType
+ tradeState := result.TradeState
+
+ if tradeState == consts.TradeStatusSuccess && tradeType == consts.TradeTypeNative {
+ slog.Info("Processing successful WeChat Pay payment")
+ err := component.ProcessPaymentSuccess(c, database.NewPaymentStore(), h.eventPub, outTradeNo, tradeNo)
+ if err != nil {
+ slog.Error("process payment success fail", slog.Any("err", err))
+ c.JSON(http.StatusOK, gin.H{"code": gopay.FAIL, "message": "failed"})
+ return
+ }
+ c.JSON(http.StatusOK, gin.H{"code": gopay.SUCCESS, "message": "Success"})
+ } else {
+ c.JSON(http.StatusOK, gin.H{"code": gopay.FAIL, "message": "Success"})
+ }
+}
+
+func NewWechatNotifyHandler(config *config.Config) (*WechatPayNotifyHandler, error) {
+ return &WechatPayNotifyHandler{
+ eventPub: &event.DefaultEventPublisher,
+ }, nil
+}
+
+
+
package router
+
+import (
+ "errors"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/payment/handler"
+)
+
+func NewPaymentRouter(config *config.Config) (*gin.Engine, error) {
+ ah, e1 := handler.NewAliPayNotifyHandler(config)
+ if e1 != nil {
+ return nil, errors.New("failed to create AliPayNotifyHandler: " + e1.Error())
+ }
+
+ wh, e2 := handler.NewWechatNotifyHandler(config)
+ if e2 != nil {
+ return nil, errors.New("failed to create WechatNotifyHandler: " + e2.Error())
+ }
+
+ ph, e3 := handler.NewPaymentHandler(config)
+ if e3 != nil {
+ return nil, errors.New("failed to create PaymentHandler: " + e3.Error())
+ }
+
+ r := gin.New()
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+ r.Use(middleware.Authenticator(config))
+
+ apiGroup := r.Group("/api/v1/payment")
+ {
+ apiGroup.POST("/wechat/notify", wh.WechatPayNotify)
+ apiGroup.POST("/alipay/notify", ah.AliPayNotify)
+ apiGroup.POST("/simple-payments", ph.CreateSimplePayment)
+ }
+
+ return r, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "path"
+ "strconv"
+ "strings"
+
+ corev1 "k8s.io/api/core/v1"
+ "k8s.io/apimachinery/pkg/api/resource"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ v1 "knative.dev/serving/pkg/apis/serving/v1"
+ "opencsg.com/csghub-server/builder/deploy/cluster"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var (
+ KeyDeployID string = "deploy_id"
+ KeyDeployType string = "deploy_type"
+ KeyUserID string = "user_id"
+ KeyDeploySKU string = "deploy_sku"
+ KeyOrderDetailID string = "order-detail-id"
+)
+
+type serviceComponentImpl struct {
+ k8sNameSpace string
+ env *config.Config
+ spaceDockerRegBase string
+ modelDockerRegBase string
+ imagePullSecret string
+}
+
+type ServiceComponent interface {
+ GenerateService(ctx context.Context, cluster cluster.Cluster, request types.SVCRequest, srvName string) (*v1.Service, error)
+ // get secret from k8s
+ // notes: admin should create nim secret "ngc-secret" and "nvidia-nim-secrets" in related namespace before deploy
+ GetNimSecret(ctx context.Context, cluster cluster.Cluster) (string, error)
+ GetServicePodsWithStatus(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string) ([]types.Instance, error)
+ // NewPersistentVolumeClaim creates a new k8s PVC with some default values set.
+ NewPersistentVolumeClaim(name string, ctx context.Context, cluster cluster.Cluster, hardware types.HardWare) error
+}
+
+func NewServiceComponent(config *config.Config, k8sNameSpace string) ServiceComponent {
+ sc := &serviceComponentImpl{
+ k8sNameSpace: k8sNameSpace,
+ env: config,
+ spaceDockerRegBase: config.Space.DockerRegBase,
+ modelDockerRegBase: config.Model.DockerRegBase,
+ imagePullSecret: config.Space.ImagePullSecret,
+ }
+ return sc
+}
+
+func (s *serviceComponentImpl) GenerateService(ctx context.Context, cluster cluster.Cluster, request types.SVCRequest, srvName string) (*v1.Service, error) {
+ annotations := request.Annotation
+
+ environments := []corev1.EnvVar{}
+ appPort := 0
+ hardware := request.Hardware
+ resReq, nodeSelector := GenerateResources(hardware)
+ var err error
+
+ if request.Env != nil {
+ // generate env
+ for key, value := range request.Env {
+ environments = append(environments, corev1.EnvVar{Name: key, Value: value})
+ }
+
+ // get app expose port from env with key=port
+ val, ok := request.Env["port"]
+ if !ok {
+ return nil, fmt.Errorf("failed to find port from env")
+ }
+
+ appPort, err = strconv.Atoi(val)
+ if err != nil {
+ return nil, fmt.Errorf("port is not valid number, error: %w", err)
+ }
+ }
+
+ // fix no gpu request case
+ if hardware.Gpu.ResourceName == "" || hardware.Gpu.Num == "" {
+ environments = append(environments, corev1.EnvVar{Name: "NVIDIA_VISIBLE_DEVICES", Value: "none"})
+ }
+ if hardware.Npu.ResourceName == "" || hardware.Npu.Num == "" {
+ environments = append(environments, corev1.EnvVar{Name: "ASCEND_VISIBLE_DEVICES", Value: "none"})
+ }
+
+ if appPort == 0 {
+ return nil, fmt.Errorf("app export port is not defined")
+ }
+
+ // knative service spec container port
+ exposePorts := []corev1.ContainerPort{{
+ ContainerPort: int32(appPort),
+ }}
+ // knative service spec resource requirement
+ resources := corev1.ResourceRequirements{
+ Limits: resReq,
+ Requests: resReq,
+ }
+
+ annotations[KeyDeployID] = strconv.FormatInt(request.DeployID, 10)
+ annotations[KeyDeployType] = strconv.Itoa(request.DeployType)
+ annotations[KeyUserID] = request.UserID
+ annotations[KeyDeploySKU] = request.Sku
+ annotations[KeyOrderDetailID] = strconv.FormatInt(request.OrderDetailID, 10)
+
+ containerImg := request.ImageID
+ // add prefix if image is not full path
+ if !strings.Contains(containerImg, "/") {
+ if request.RepoType == string(types.ModelRepo) {
+ // choose registry
+ containerImg = path.Join(s.modelDockerRegBase, request.ImageID)
+ } else if request.RepoType == string(types.SpaceRepo) {
+ // choose registry
+ containerImg = path.Join(s.spaceDockerRegBase, request.ImageID)
+ }
+ }
+
+ templateAnnotations := make(map[string]string)
+ if request.RepoType == string(types.ModelRepo) {
+ // auto scaling
+ templateAnnotations["autoscaling.knative.dev/class"] = "kpa.autoscaling.knative.dev"
+ templateAnnotations["enable-scale-to-zero"] = "false"
+ templateAnnotations["autoscaling.knative.dev/metric"] = "concurrency"
+ templateAnnotations["autoscaling.knative.dev/target"] = "5"
+ templateAnnotations["autoscaling.knative.dev/target-utilization-percentage"] = "90"
+ templateAnnotations["autoscaling.knative.dev/min-scale"] = strconv.Itoa(request.MinReplica)
+ templateAnnotations["autoscaling.knative.dev/max-scale"] = strconv.Itoa(request.MaxReplica)
+ templateAnnotations["serving.knative.dev/progress-deadline"] = fmt.Sprintf("%dm", s.env.Model.DeployTimeoutInMin)
+ }
+ initialDelaySeconds := 10
+ periodSeconds := 10
+ failureThreshold := 3
+ if request.DeployType == types.InferenceType {
+ initialDelaySeconds = s.env.Space.ReadnessDelaySeconds
+ periodSeconds = s.env.Space.ReadnessPeriodSeconds
+ failureThreshold = s.env.Space.ReadnessFailureThreshold
+ }
+
+ imagePullSecrets := []corev1.LocalObjectReference{
+ {
+ Name: s.imagePullSecret,
+ },
+ }
+
+ // handle nim engine
+ if strings.Contains(containerImg, "nvcr.io/nim/") {
+ imagePullSecrets = append(imagePullSecrets, corev1.LocalObjectReference{
+ Name: s.env.Model.NimDockerSecretName,
+ })
+ ngc_api_key, err := s.GetNimSecret(ctx, cluster)
+ if err != nil {
+ return nil, fmt.Errorf("can not find secret %s in %s namespace , error: %w", s.env.Model.NimNGCSecretName, s.k8sNameSpace, err)
+ }
+ environments = append(environments, corev1.EnvVar{Name: "NGC_API_KEY", Value: ngc_api_key})
+ environments = append(environments, corev1.EnvVar{Name: "NIM_CACHE_PATH", Value: "/workspace"})
+ }
+
+ service := &v1.Service{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: srvName,
+ Namespace: s.k8sNameSpace,
+ Annotations: annotations,
+ },
+ Spec: v1.ServiceSpec{
+ ConfigurationSpec: v1.ConfigurationSpec{
+ Template: v1.RevisionTemplateSpec{
+ ObjectMeta: metav1.ObjectMeta{
+ Annotations: templateAnnotations,
+ },
+ Spec: v1.RevisionSpec{
+ PodSpec: corev1.PodSpec{
+ NodeSelector: nodeSelector,
+ Containers: []corev1.Container{{
+ // TODO: docker registry url + image id
+ // Image: "ghcr.io/knative/helloworld-go:latest",
+ Image: containerImg,
+ Ports: exposePorts,
+ Resources: resources,
+ Env: environments,
+ ReadinessProbe: &corev1.Probe{
+ InitialDelaySeconds: int32(initialDelaySeconds),
+ PeriodSeconds: int32(periodSeconds),
+ FailureThreshold: int32(failureThreshold),
+ },
+ }},
+ ImagePullSecrets: imagePullSecrets,
+ },
+ },
+ },
+ },
+ },
+ }
+ return service, nil
+}
+
+// get secret from k8s
+// notes: admin should create nim secret "ngc-secret" and "nvidia-nim-secrets" in related namespace before deploy
+func (s *serviceComponentImpl) GetNimSecret(ctx context.Context, cluster cluster.Cluster) (string, error) {
+ secret, err := cluster.Client.CoreV1().Secrets(s.k8sNameSpace).Get(ctx, s.env.Model.NimNGCSecretName, metav1.GetOptions{})
+ if err != nil {
+ return "", err
+ }
+ return string(secret.Data["NGC_API_KEY"]), nil
+}
+
+func (s *serviceComponentImpl) GetServicePodsWithStatus(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string) ([]types.Instance, error) {
+ labelSelector := fmt.Sprintf("serving.knative.dev/service=%s", srvName)
+ // Get the list of Pods based on the label selector
+ pods, err := cluster.Client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+ LabelSelector: labelSelector,
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ // Extract the Pod names and status
+ var podInstances []types.Instance
+ for _, pod := range pods.Items {
+ podInstances = append(podInstances,
+ types.Instance{
+ Name: pod.Name,
+ Status: string(pod.Status.Phase),
+ },
+ )
+ slog.Debug("pod", slog.Any("pod.Name", pod.Name), slog.Any("pod.Status.Phase", pod.Status.Phase))
+ }
+ return podInstances, nil
+}
+
+func GenerateResources(hardware types.HardWare) (map[corev1.ResourceName]resource.Quantity, map[string]string) {
+ nodeSelector := make(map[string]string)
+ resReq := make(map[corev1.ResourceName]resource.Quantity)
+
+ // generate node selector
+ if hardware.Gpu.Labels != nil {
+ for key, value := range hardware.Gpu.Labels {
+ nodeSelector[key] = value
+ }
+ }
+ if hardware.Cpu.Labels != nil {
+ for key, value := range hardware.Cpu.Labels {
+ nodeSelector[key] = value
+ }
+ }
+
+ // generate knative resource requirement
+ if hardware.Cpu.Num != "" {
+ resReq[corev1.ResourceCPU] = resource.MustParse(hardware.Cpu.Num)
+ }
+ if hardware.Memory != "" {
+ resReq[corev1.ResourceMemory] = resource.MustParse(hardware.Memory)
+ }
+ if hardware.EphemeralStorage != "" {
+ resReq[corev1.ResourceEphemeralStorage] = resource.MustParse(hardware.EphemeralStorage)
+ }
+ if hardware.Gpu.ResourceName != "" && hardware.Gpu.Num != "" {
+ resReq[corev1.ResourceName(hardware.Gpu.ResourceName)] = resource.MustParse(hardware.Gpu.Num)
+ }
+ if hardware.Npu.ResourceName != "" && hardware.Npu.Num != "" {
+ resReq[corev1.ResourceName(hardware.Npu.ResourceName)] = resource.MustParse(hardware.Npu.Num)
+ }
+ return resReq, nodeSelector
+}
+
+// NewPersistentVolumeClaim creates a new k8s PVC with some default values set.
+func (s *serviceComponentImpl) NewPersistentVolumeClaim(name string, ctx context.Context, cluster cluster.Cluster, hardware types.HardWare) error {
+ // Check if it already exists
+ _, err := cluster.Client.CoreV1().PersistentVolumeClaims(s.k8sNameSpace).Get(ctx, name, metav1.GetOptions{})
+ if err == nil {
+ return nil
+ }
+
+ storageSize := hardware.EphemeralStorage
+ if storageSize == "" {
+ storageSize = "50Gi"
+ }
+
+ storage, err := resource.ParseQuantity(storageSize)
+ if err != nil {
+ return err
+ }
+ pvc := corev1.PersistentVolumeClaim{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: name,
+ Namespace: s.k8sNameSpace,
+ },
+ Spec: corev1.PersistentVolumeClaimSpec{
+ AccessModes: []corev1.PersistentVolumeAccessMode{
+ corev1.ReadWriteOnce,
+ },
+ Resources: corev1.VolumeResourceRequirements{
+ Requests: corev1.ResourceList{
+ corev1.ResourceStorage: storage,
+ },
+ },
+ StorageClassName: &cluster.StorageClass,
+ },
+ }
+ _, err = cluster.Client.CoreV1().PersistentVolumeClaims(s.k8sNameSpace).Create(ctx, &pvc, metav1.CreateOptions{})
+ return err
+}
+
+
+
package component
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "log/slog"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
+ "github.com/argoproj/argo-workflows/v3/pkg/client/informers/externalversions"
+ "github.com/google/uuid"
+ corev1 "k8s.io/api/core/v1"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/util/runtime"
+ "k8s.io/client-go/tools/cache"
+ "k8s.io/utils/ptr"
+ "opencsg.com/csghub-server/builder/deploy/cluster"
+ "opencsg.com/csghub-server/builder/event"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type workFlowComponentImpl struct {
+ config *config.Config
+ wf database.ArgoWorkFlowStore
+ clusterPool *cluster.ClusterPool
+ eventPub *event.EventPublisher
+}
+
+type WorkFlowComponent interface {
+ // Create workflow
+ CreateWorkflow(ctx context.Context, req types.ArgoWorkFlowReq) (*database.ArgoWorkflow, error)
+ // Update workflow
+ UpdateWorkflow(ctx context.Context, update *v1alpha1.Workflow) (*database.ArgoWorkflow, error)
+ // find workflow by user name
+ FindWorkFlows(ctx context.Context, username string, per, page int) ([]database.ArgoWorkflow, int, error)
+ // generate workflow templates
+ DeleteWorkflow(ctx context.Context, id int64, username string) error
+ GetWorkflow(ctx context.Context, id int64, username string) (*database.ArgoWorkflow, error)
+ DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error
+ FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error)
+ RunWorkflowsInformer(clusterPool *cluster.ClusterPool, config *config.Config)
+ StartAcctRequestFee(wf database.ArgoWorkflow)
+}
+
+func NewWorkFlowComponent(config *config.Config, clusterPool *cluster.ClusterPool) WorkFlowComponent {
+ wf := database.NewArgoWorkFlowStore()
+ wc := &workFlowComponentImpl{
+ config: config,
+ wf: wf,
+ clusterPool: clusterPool,
+ eventPub: &event.DefaultEventPublisher,
+ }
+ //watch workflows events
+ go wc.RunWorkflowsInformer(clusterPool, config)
+ return wc
+}
+
+// Create workflow
+func (wc *workFlowComponentImpl) CreateWorkflow(ctx context.Context, req types.ArgoWorkFlowReq) (*database.ArgoWorkflow, error) {
+ // create workflow in db
+ namespace := wc.config.Argo.Namespace
+ if req.ShareMode {
+ namespace = wc.config.Argo.QuotaNamespace
+ }
+ cluster, clusterId, err := GetCluster(ctx, wc.clusterPool, req.ClusterID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get cluster by id: %v", err)
+ }
+ argowf := &database.ArgoWorkflow{
+ Username: req.Username,
+ UserUUID: req.UserUUID,
+ TaskName: req.TaskName,
+ TaskId: req.TaskId,
+ TaskType: req.TaskType,
+ RepoIds: req.RepoIds,
+ TaskDesc: req.TaskDesc,
+ Image: req.Image,
+ Datasets: req.Datasets,
+ ResourceId: req.ResourceId,
+ ResourceName: req.ResourceName,
+ ClusterID: clusterId,
+ RepoType: req.RepoType,
+ Namespace: namespace,
+ Status: v1alpha1.WorkflowPhase(v1alpha1.NodePending),
+ }
+ // create workflow in argo
+ awf := generateWorkflow(req, wc.config)
+
+ _, err = cluster.ArgoClient.ArgoprojV1alpha1().Workflows(namespace).Create(ctx, awf, v1.CreateOptions{})
+ if err != nil {
+ return nil, fmt.Errorf("failed to create workflow in argo: %v", err)
+ }
+ wf, err := wc.wf.CreateWorkFlow(ctx, *argowf)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create workflow in db: %v", err)
+ }
+ return wf, nil
+}
+
+func (wc *workFlowComponentImpl) DeleteWorkflow(ctx context.Context, id int64, username string) error {
+ wf, err := wc.FindWorkFlowById(ctx, id)
+ if err != nil {
+ return fmt.Errorf("failed to get workflow by id: %v", err)
+ }
+ if wf.Username != username {
+ return fmt.Errorf("no permission to delete workflow")
+ }
+ cluster, _, err := GetCluster(ctx, wc.clusterPool, wf.ClusterID)
+ if err != nil {
+ return fmt.Errorf("failed to get cluster by id: %v", err)
+ }
+ err = cluster.ArgoClient.ArgoprojV1alpha1().Workflows(wf.Namespace).Delete(ctx, wf.TaskId, v1.DeleteOptions{})
+ if err != nil {
+ slog.Warn("Error deleting argo workflow", slog.Any("error", err))
+ }
+ return wc.wf.DeleteWorkFlow(ctx, wf.ID)
+}
+
+func (wc *workFlowComponentImpl) GetWorkflow(ctx context.Context, id int64, username string) (*database.ArgoWorkflow, error) {
+ wf, err := wc.FindWorkFlowById(ctx, id)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get workflow by id: %v", err)
+ }
+ if wf.Username != username {
+ return nil, fmt.Errorf("no permission to get workflow")
+ }
+ return &wf, nil
+}
+
+// Update workflow
+func (wc *workFlowComponentImpl) UpdateWorkflow(ctx context.Context, update *v1alpha1.Workflow) (*database.ArgoWorkflow, error) {
+ oldwf, err := wc.wf.FindByTaskID(ctx, update.Name)
+ if err != nil {
+ return nil, err
+ }
+
+ oldwf.Reason = update.Status.Message
+ if node, ok := update.Status.Nodes[oldwf.TaskId]; ok {
+ oldwf.Status = v1alpha1.WorkflowPhase(node.Phase)
+ if node.Phase == v1alpha1.NodeRunning {
+ oldwf.StartTime = time.Now()
+ }
+ if _, exists := types.WorkFlowFinished[oldwf.Status]; exists {
+ oldwf.EndTime = time.Now()
+ }
+ if node.Outputs != nil && node.Outputs.Parameters != nil {
+ for _, output := range node.Outputs.Parameters {
+ if output.Name == "result" && output.Value != nil {
+ result := strings.Split(output.Value.String(), ",")
+ oldwf.ResultURL = result[0]
+ oldwf.DownloadURL = result[1]
+ wc.StartAcctRequestFee(oldwf)
+ break
+ }
+ }
+ }
+ }
+ return wc.wf.UpdateWorkFlow(ctx, oldwf)
+}
+
+// DeleteWorkflowInargo
+func (wc *workFlowComponentImpl) DeleteWorkflowInargo(ctx context.Context, delete *v1alpha1.Workflow) error {
+ wf, err := wc.wf.FindByTaskID(ctx, delete.Name)
+ if err != nil {
+ return fmt.Errorf("failed to get workflow by id: %v", err)
+ }
+ // for deleted case,check if the workflow did not finish
+ if wf.Status == v1alpha1.WorkflowPending || wf.Status == v1alpha1.WorkflowRunning {
+ wf.Status = v1alpha1.WorkflowFailed
+ wf.Reason = "deleted by admin"
+ _, err = wc.wf.UpdateWorkFlow(ctx, wf)
+ return err
+ }
+ return nil
+}
+
+func (wc *workFlowComponentImpl) FindWorkFlowById(ctx context.Context, id int64) (database.ArgoWorkflow, error) {
+ return wc.wf.FindByID(ctx, id)
+}
+
+// find workflow by user name
+func (wc *workFlowComponentImpl) FindWorkFlows(ctx context.Context, username string, per, page int) ([]database.ArgoWorkflow, int, error) {
+ return wc.wf.FindByUsername(ctx, username, per, page)
+}
+
+// create workflow in argo
+func generateWorkflow(req types.ArgoWorkFlowReq, config *config.Config) *v1alpha1.Workflow {
+ templates := []v1alpha1.Template{}
+ for _, v := range req.Templates {
+ resReq, _ := GenerateResources(v.HardWare)
+ environments := []corev1.EnvVar{}
+ for key, value := range v.Env {
+ environments = append(environments, corev1.EnvVar{Name: key, Value: value})
+ }
+ environments = append(environments, corev1.EnvVar{Name: "S3_ACCESS_ID", Value: config.S3.AccessKeyID})
+ environments = append(environments, corev1.EnvVar{Name: "S3_ACCESS_SECRET", Value: config.S3.AccessKeySecret})
+ environments = append(environments, corev1.EnvVar{Name: "S3_BUCKET", Value: config.Argo.S3PublicBucket})
+ environments = append(environments, corev1.EnvVar{Name: "S3_ENDPOINT", Value: config.S3.Endpoint})
+ environments = append(environments, corev1.EnvVar{Name: "S3_SSL_ENABLED", Value: strconv.FormatBool(config.S3.EnableSSL)})
+ // fix no gpu request case
+ if v.HardWare.Gpu.ResourceName == "" || v.HardWare.Gpu.Num == "" {
+ environments = append(environments, corev1.EnvVar{Name: "NVIDIA_VISIBLE_DEVICES", Value: "none"})
+ }
+ if v.HardWare.Npu.ResourceName == "" || v.HardWare.Npu.Num == "" {
+ environments = append(environments, corev1.EnvVar{Name: "ASCEND_VISIBLE_DEVICES", Value: "none"})
+ }
+
+ resources := corev1.ResourceRequirements{
+ Limits: resReq,
+ Requests: resReq,
+ }
+
+ containerImg := v.Image
+ // add prefix if image is not full path
+ if !strings.Contains(containerImg, "/") {
+ if req.RepoType == string(types.ModelRepo) {
+ // choose registry
+ containerImg = path.Join(config.Model.DockerRegBase, v.Image)
+ } else if req.RepoType == string(types.SpaceRepo) {
+ // choose registry
+ containerImg = path.Join(config.Space.DockerRegBase, v.Image)
+ }
+ }
+
+ templates = append(templates, v1alpha1.Template{
+ Name: v.Name,
+ //NodeSelector: nodeSelector,
+ Container: &corev1.Container{
+ Image: containerImg,
+ Command: v.Command,
+ Env: environments,
+ Args: v.Args,
+ Resources: resources,
+ },
+ Outputs: v1alpha1.Outputs{
+ Parameters: []v1alpha1.Parameter{
+ {
+ Name: "result",
+ ValueFrom: &v1alpha1.ValueFrom{
+ Path: "/tmp/output.txt",
+ },
+ },
+ },
+ },
+ })
+ }
+
+ workflowObject := &v1alpha1.Workflow{
+ ObjectMeta: v1.ObjectMeta{
+ Name: req.TaskId,
+ Labels: map[string]string{
+ "workflow-scope": "csghub",
+ },
+ },
+ Spec: v1alpha1.WorkflowSpec{
+ Priority: ptr.To(int32(3)),
+ ServiceAccountName: config.Argo.ServiceAccountName,
+ Templates: templates,
+ Entrypoint: req.Entrypoint,
+ TTLStrategy: &v1alpha1.TTLStrategy{
+ // Set TTL here
+ SecondsAfterCompletion: ptr.To(int32(config.Argo.JobTTL)),
+ },
+ },
+ }
+
+ return workflowObject
+}
+
+func (wc *workFlowComponentImpl) RunWorkflowsInformer(clusterPool *cluster.ClusterPool, c *config.Config) {
+ clientset := clusterPool.Clusters[0].ArgoClient
+ f := externalversions.NewSharedInformerFactoryWithOptions(clientset, 60*time.Second, externalversions.WithTweakListOptions(func(list *v1.ListOptions) {
+ list.LabelSelector = "workflow-scope=csghub"
+ }))
+ informer := f.Argoproj().V1alpha1().Workflows().Informer()
+
+ stopper := make(chan struct{})
+ defer close(stopper)
+
+ defer runtime.HandleCrash()
+
+ eventHandler := cache.ResourceEventHandlerFuncs{
+ AddFunc: func(obj interface{}) {
+ // triggered in startup
+ wf := obj.(*v1alpha1.Workflow)
+ bg, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ _, err := wc.UpdateWorkflow(bg, wf)
+ if err != nil {
+ slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", wf.Name))
+ }
+ },
+ UpdateFunc: func(oldObj, newObj interface{}) {
+ oldWF := oldObj.(*v1alpha1.Workflow)
+ newWF := newObj.(*v1alpha1.Workflow)
+
+ // compare status
+ bg, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if newWF.Status.Nodes == nil || oldWF.Status.Nodes == nil {
+ return
+ }
+ if oldWF.Status.Nodes[oldWF.Name].Phase != newWF.Status.Nodes[oldWF.Name].Phase {
+ _, err := wc.UpdateWorkflow(bg, newWF)
+ if err != nil {
+ slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", newWF.Name))
+ }
+ }
+ },
+ DeleteFunc: func(obj interface{}) {
+ //handle some special case
+ wf := obj.(*v1alpha1.Workflow)
+ bg, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ err := wc.DeleteWorkflowInargo(bg, wf)
+ if err != nil {
+ slog.Error("fail to update workflow", slog.Any("error", err), slog.Any("job id", wf.Name))
+ }
+ },
+ }
+
+ _, _ = informer.AddEventHandler(eventHandler)
+
+ informer.Run(stopper)
+ if !cache.WaitForCacheSync(stopper, informer.HasSynced) {
+ runtime.HandleError(fmt.Errorf("timed out waiting for caches to sync"))
+ return
+ }
+}
+
+func (wc *workFlowComponentImpl) StartAcctRequestFee(wf database.ArgoWorkflow) {
+ if !wc.config.IsMasterHost {
+ return
+ }
+ if wf.ResourceId == 0 {
+ return
+ }
+ duration := wf.EndTime.Sub(wf.StartTime)
+ minutes := duration.Minutes()
+ if minutes < 1 {
+ return
+ }
+ slog.Info("start to acct request fee", slog.Any("mins", minutes))
+ event := types.METERING_EVENT{
+ Uuid: uuid.New(),
+ UserUUID: wf.UserUUID,
+ Value: int64(minutes),
+ ValueType: types.TimeDurationMinType,
+ Scene: int(types.SceneEvaluation),
+ OpUID: "",
+ ResourceID: strconv.FormatInt(wf.ResourceId, 10),
+ ResourceName: wf.ResourceName,
+ CustomerID: wf.TaskId,
+ CreatedAt: time.Now(),
+ }
+ str, err := json.Marshal(event)
+ if err != nil {
+ slog.Error("error marshal metering event", slog.Any("event", event), slog.Any("error", err))
+ return
+ }
+ err = wc.eventPub.PublishMeteringEvent(str)
+ if err != nil {
+ slog.Error("failed to pub metering event", slog.Any("data", string(str)), slog.Any("error", err))
+ } else {
+ slog.Info("pub metering event success", slog.Any("data", string(str)))
+ }
+}
+
+// get cluster
+func GetCluster(ctx context.Context, clusterPool *cluster.ClusterPool, clusterID string) (*cluster.Cluster, string, error) {
+ if clusterID == "" {
+ clusterInfo, err := clusterPool.ClusterStore.ByClusterConfig(ctx, clusterPool.Clusters[0].ID)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to get cluster info: %v", err)
+ }
+ return &clusterPool.Clusters[0], clusterInfo.ClusterID, nil
+ }
+ cluster, err := clusterPool.GetClusterByID(ctx, clusterID)
+ if err != nil {
+ return nil, clusterID, fmt.Errorf("failed to get cluster by id: %v", err)
+ }
+ return cluster, clusterID, nil
+}
+
+
+
package handler
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "path"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ corev1 "k8s.io/api/core/v1"
+ k8serrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ v1 "knative.dev/serving/pkg/apis/serving/v1"
+ "opencsg.com/csghub-server/builder/deploy/cluster"
+ "opencsg.com/csghub-server/builder/deploy/common"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/runner/component"
+)
+
+type K8sHandler struct {
+ clusterPool *cluster.ClusterPool
+ k8sNameSpace string
+ modelDockerRegBase string
+ env *config.Config
+ s component.ServiceComponent
+}
+
+func NewK8sHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*K8sHandler, error) {
+ domainParts := strings.SplitN(config.Space.InternalRootDomain, ".", 2)
+ serviceComponent := component.NewServiceComponent(config, domainParts[0])
+ return &K8sHandler{
+ k8sNameSpace: domainParts[0],
+ clusterPool: clusterPool,
+ env: config,
+ s: serviceComponent,
+ modelDockerRegBase: config.Model.DockerRegBase,
+ }, nil
+}
+
+func (s *K8sHandler) RunService(c *gin.Context) {
+ request := &types.SVCRequest{}
+ err := c.BindJSON(&request)
+ if err != nil {
+ slog.Error("runService get bad request", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ slog.Debug("Recv request", slog.Any("body", request))
+
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster ", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ srvName := s.getServiceNameFromRequest(c)
+ // check if the ksvc exists
+ _, err = cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err == nil {
+ err = s.removeServiceForcely(c, cluster, srvName)
+ if err != nil {
+ slog.Error("fail to remove service", slog.Any("error", err), slog.Any("req", request))
+ }
+ slog.Info("service already exists,delete it first", slog.String("srv_name", srvName), slog.Any("image_id", request.ImageID))
+ }
+ service, err := s.s.GenerateService(c, *cluster, *request, srvName)
+ if err != nil {
+ slog.Error("fail to generate service ", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ volumes := []corev1.Volume{}
+ volumeMounts := []corev1.VolumeMount{}
+ if request.DeployType != types.SpaceType {
+ // dshm volume for multi-gpu share memory
+ volumes = append(volumes, corev1.Volume{
+ Name: "dshm",
+ VolumeSource: corev1.VolumeSource{
+ EmptyDir: &corev1.EmptyDirVolumeSource{
+ Medium: corev1.StorageMediumMemory,
+ },
+ },
+ })
+
+ volumeMounts = append(volumeMounts, corev1.VolumeMount{
+ Name: "dshm",
+ MountPath: "/dev/shm",
+ })
+ }
+ pvcName := srvName
+ if request.DeployType == types.InferenceType {
+ pvcName = request.UserID
+ }
+ // add pvc if possible
+ // space image was built from user's code, model cache dir is hard to control
+ // so no PV cache for space case so far
+ if cluster.StorageClass != "" && request.DeployType != types.SpaceType {
+ err = s.s.NewPersistentVolumeClaim(pvcName, c, *cluster, request.Hardware)
+ if err != nil {
+ slog.Error("Failed to create persist volume", "error", err)
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create persist volume"})
+ return
+ }
+ volumes = append(volumes, corev1.Volume{
+ Name: "nas-pvc",
+ VolumeSource: corev1.VolumeSource{
+ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+ ClaimName: pvcName,
+ },
+ },
+ })
+
+ volumeMounts = append(volumeMounts, corev1.VolumeMount{
+ Name: "nas-pvc",
+ MountPath: "/workspace",
+ })
+ }
+ service.Spec.Template.Spec.Volumes = volumes
+ service.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
+
+ slog.Debug("ksvc", slog.Any("knative service", service))
+
+ // create ksvc
+ _, err = cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Create(c, service, metav1.CreateOptions{})
+ if err != nil {
+ slog.Error("Failed to create service", "error", err, slog.Int64("deploy_id", request.DeployID),
+ slog.String("image_id", request.ImageID),
+ slog.String("srv_name", srvName))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create service"})
+ return
+ }
+
+ slog.Info("service created successfully", slog.String("srv_name", srvName), slog.Int64("deploy_id", request.DeployID))
+ c.JSON(http.StatusOK, gin.H{"message": "Service created successfully"})
+}
+
+func (s *K8sHandler) StopService(c *gin.Context) {
+ var resp types.StopResponse
+ var request = &types.StopRequest{}
+ err := c.BindJSON(request)
+
+ if err != nil {
+ slog.Error("stopService get bad request", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster ", slog.Any("error", err))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ srvName := s.getServiceNameFromRequest(c)
+ srv, err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).
+ Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err != nil {
+ k8serr := new(k8serrors.StatusError)
+ if errors.As(err, &k8serr) {
+ if k8serr.Status().Code == http.StatusNotFound {
+ slog.Info("stop image skip,service not exist", slog.String("srv_name", srvName), slog.Any("k8s_err", k8serr))
+ resp.Code = 0
+ resp.Message = "skip,service not exist"
+ c.JSON(http.StatusOK, nil)
+ return
+ }
+ }
+ slog.Error("stop image failed, cannot get service info", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ resp.Code = -1
+ resp.Message = "failed to get service status"
+ c.JSON(http.StatusInternalServerError, resp)
+ return
+ }
+
+ if srv == nil {
+ resp.Code = 0
+ resp.Message = "service not exist"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+ err = s.removeServiceForcely(c, cluster, srvName)
+ if err != nil {
+ slog.Error("stop image failed, cannot delete service ", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ resp.Code = -1
+ resp.Message = "failed to get service status"
+ c.JSON(http.StatusInternalServerError, resp)
+ return
+ }
+
+ slog.Info("service deleted", slog.String("srv_name", srvName))
+ resp.Code = 0
+ resp.Message = "service deleted"
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) removeServiceForcely(c *gin.Context, cluster *cluster.Cluster, svcName string) error {
+ err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Delete(context.Background(), svcName, *metav1.NewDeleteOptions(0))
+ if err != nil {
+ return err
+ }
+ podNames, _ := s.GetServicePods(c.Request.Context(), *cluster, svcName, s.k8sNameSpace, -1)
+ if podNames == nil {
+ return nil
+ }
+ //before k8s 1.31, kill pod does not kill the process immediately, instead we still need wait for the process to exit. more details see: https://github.com/kubernetes/kubernetes/issues/120449
+ gracePeriodSeconds := int64(10)
+ deletePolicy := metav1.DeletePropagationForeground
+ deleteOptions := metav1.DeleteOptions{
+ GracePeriodSeconds: &gracePeriodSeconds,
+ PropagationPolicy: &deletePolicy,
+ }
+
+ for _, podName := range podNames {
+ errForce := cluster.Client.CoreV1().Pods(s.k8sNameSpace).Delete(c.Request.Context(), podName, deleteOptions)
+ if errForce != nil {
+ slog.Error("removeServiceForcely failed to delete pod", slog.String("pod_name", podName), slog.Any("error", errForce))
+ }
+ }
+ return nil
+}
+
+func (s *K8sHandler) UpdateService(c *gin.Context) {
+ var resp types.ModelUpdateResponse
+ var request = &types.ModelUpdateRequest{}
+ err := c.BindJSON(request)
+
+ if err != nil {
+ slog.Error("updateService get bad request", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster ", slog.Any("error", err))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ srvName := s.getServiceNameFromRequest(c)
+ srv, err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).
+ Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err != nil {
+ k8serr := new(k8serrors.StatusError)
+ if errors.As(err, &k8serr) {
+ if k8serr.Status().Code == http.StatusNotFound {
+ slog.Info("update service skip,service not exist", slog.String("srv_name", srvName), slog.Any("k8s_err", k8serr))
+ resp.Code = 0
+ resp.Message = "skip,service not exist"
+ c.JSON(http.StatusOK, nil)
+ return
+ }
+ }
+ slog.Error("update service failed, cannot get service info", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ resp.Code = -1
+ resp.Message = "failed to get service status"
+ c.JSON(http.StatusInternalServerError, resp)
+ return
+ }
+
+ if srv == nil {
+ resp.Code = 0
+ resp.Message = "service not exist"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+ // Update Image
+ containerImg := path.Join(s.modelDockerRegBase, request.ImageID)
+ srv.Spec.Template.Spec.Containers[0].Image = containerImg
+ // Update env
+ environments := []corev1.EnvVar{}
+ if request.Env != nil {
+ // generate env
+ for key, value := range request.Env {
+ environments = append(environments, corev1.EnvVar{Name: key, Value: value})
+ }
+ srv.Spec.Template.Spec.Containers[0].Env = environments
+ }
+ // Update CPU and Memory requests and limits
+ hardware := request.Hardware
+ resReq, _ := component.GenerateResources(hardware)
+ resources := corev1.ResourceRequirements{
+ Limits: resReq,
+ Requests: resReq,
+ }
+ srv.Spec.Template.Spec.Containers[0].Resources = resources
+ // Update replica
+ srv.Spec.Template.Annotations["autoscaling.knative.dev/min-scale"] = strconv.Itoa(request.MinReplica)
+ srv.Spec.Template.Annotations["autoscaling.knative.dev/max-scale"] = strconv.Itoa(request.MaxReplica)
+
+ _, err = cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Update(c, srv, metav1.UpdateOptions{})
+ if err != nil {
+ slog.Error("failed to update service ", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ resp.Code = -1
+ resp.Message = "failed to update service"
+ c.JSON(http.StatusInternalServerError, resp)
+ return
+ }
+
+ slog.Info("service updated", slog.String("srv_name", srvName))
+ resp.Code = 0
+ resp.Message = "service updated"
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) ServiceStatus(c *gin.Context) {
+ var resp types.StatusResponse
+
+ var request = &types.StatusRequest{}
+ err := c.BindJSON(request)
+
+ if err != nil {
+ slog.Error("serviceStatus get bad request", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+
+ if err != nil {
+ slog.Error("fail to get cluster ", slog.Any("error", err))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ srvName := s.getServiceNameFromRequest(c)
+ srv, err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).
+ Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err != nil {
+ slog.Error("get image status failed, cannot get service info", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ resp.Code = common.Stopped
+ resp.Message = "failed to get service status"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+ deployIDStr := srv.Annotations["deploy_id"]
+ deployID, _ := strconv.ParseInt(deployIDStr, 10, 64)
+ resp.DeployID = deployID
+ resp.UserID = srv.Annotations["user_id"]
+
+ // retrieve pod list and status
+ if request.NeedDetails {
+ instList, err := s.s.GetServicePodsWithStatus(c.Request.Context(), *cluster, srvName, s.k8sNameSpace)
+ if err != nil {
+ slog.Error("fail to get service pod name list", slog.Any("error", err))
+ c.JSON(http.StatusNotFound, gin.H{"error": "fail to get service pod name list"})
+ return
+ }
+ resp.Instances = instList
+ }
+
+ if srv.IsFailed() {
+ resp.Code = common.DeployFailed
+ // read message of Ready
+ resp.Message = srv.Status.GetCondition(v1.ServiceConditionReady).Message
+ // append message of ConfigurationsReady
+ srvConfigReady := srv.Status.GetCondition(v1.ServiceConditionConfigurationsReady)
+ if srvConfigReady != nil {
+ resp.Message += srvConfigReady.Message
+ }
+ // for inference case: model loading case one pod is not ready
+ for _, instance := range resp.Instances {
+ if instance.Status == string(corev1.PodRunning) || instance.Status == string(corev1.PodPending) {
+ resp.Code = common.Deploying
+ break
+ }
+ }
+ slog.Info("service status is failed", slog.String("srv_name", srvName), slog.Any("resp", resp))
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+
+ if srv.IsReady() {
+ podNames, err := s.GetServicePods(c.Request.Context(), *cluster, srvName, s.k8sNameSpace, 1)
+ if err != nil {
+ slog.Error("get image status failed, can not get pods info", slog.String("srv_name", srvName), slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"code": 0, "message": "unknown service status, failed to get pods"})
+ return
+ }
+ if len(podNames) == 0 {
+ resp.Code = common.Sleeping
+ resp.Message = "service sleeping, no running pods"
+ slog.Debug("get image status success", slog.String("srv_name", srvName), slog.Any("resp", resp))
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+
+ resp.Code = common.Running
+ resp.Message = "service running"
+ if srv.Status.URL != nil {
+ slog.Debug("knative endpoint", slog.Any("svc name", srvName), slog.Any("url", srv.Status.URL.URL().String()))
+ resp.Endpoint = srv.Status.URL.URL().String()
+ }
+
+ slog.Debug("service status is ready", slog.String("srv_name", srvName), slog.Any("resp", resp))
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+
+ // default to deploying status
+ resp.Code = common.Deploying
+ resp.Message = "service is not ready or failed"
+ slog.Info("get service status success, service is not ready or failed", slog.String("srv_name", srvName), slog.Any("resp", resp))
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) ServiceLogs(c *gin.Context) {
+ var request = &types.LogsRequest{}
+ err := c.BindJSON(request)
+
+ if err != nil {
+ slog.Error("serviceLogs get bad request", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster ", slog.Any("error", err))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ srvName := s.getServiceNameFromRequest(c)
+ podNames, err := s.GetServicePods(c.Request.Context(), *cluster, srvName, s.k8sNameSpace, 1)
+ if err != nil {
+ slog.Error("failed to read image logs, cannot get pods info", slog.Any("error", err), slog.String("srv_name", srvName))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get pods info"})
+ return
+ }
+ if len(podNames) == 0 {
+ slog.Error("failed to read image logs, no running pods", slog.String("srv_name", srvName))
+ c.JSON(http.StatusNotFound, gin.H{"error": "no running pods, service maybe sleeping"})
+ return
+ }
+ s.GetLogsByPod(c, *cluster, podNames[0], srvName)
+}
+
+func (s *K8sHandler) ServiceLogsByPod(c *gin.Context) {
+ var request = &types.ServiceRequest{}
+ err := c.BindJSON(request)
+
+ if err != nil {
+ slog.Error("serviceLogs get bad request", slog.Any("error", err), slog.Any("req", request))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster ", slog.Any("error", err))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ srvName := s.getServiceNameFromRequest(c)
+ podName := s.getPodNameFromRequest(c)
+ s.GetLogsByPod(c, *cluster, podName, srvName)
+}
+
+func (s *K8sHandler) GetLogsByPod(c *gin.Context, cluster cluster.Cluster, podName string, srvName string) {
+
+ logs := cluster.Client.CoreV1().Pods(s.k8sNameSpace).GetLogs(podName, &corev1.PodLogOptions{
+ Container: "user-container",
+ Follow: true,
+ })
+ stream, err := logs.Stream(context.Background())
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to open stream"})
+ return
+ }
+ defer stream.Close()
+
+ // c.Header("Content-Type", "text/event-stream")
+ c.Header("Content-Type", "text/plain")
+ c.Header("Cache-Control", "no-cache")
+ c.Header("Connection", "keep-alive")
+ c.Header("Transfer-Encoding", "chunked")
+ c.Writer.WriteHeader(http.StatusOK)
+ buf := make([]byte, 32*1024)
+
+ pod, err := cluster.Client.CoreV1().Pods(s.k8sNameSpace).Get(context.Background(), podName, metav1.GetOptions{})
+ if err != nil {
+ slog.Error("fail to get pod ", slog.Any("error", err), slog.String("pod name", podName))
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+
+ if pod.Status.Phase == "Pending" {
+ for _, condition := range pod.Status.Conditions {
+ if condition.Type == "PodScheduled" && condition.Status == "False" {
+ message := fmt.Sprintf("Pod is pending due to reason: %s, message: %s", condition.Reason, condition.Message)
+ _, err = c.Writer.Write([]byte(message))
+ if err != nil {
+ slog.Error("write data failed", "error", err)
+ }
+ c.Writer.Flush()
+ c.JSON(http.StatusBadRequest, gin.H{"error": message})
+ return
+ }
+ }
+ }
+
+ for {
+ select {
+ case <-c.Request.Context().Done():
+ slog.Info("logs request context done", slog.Any("error", c.Request.Context().Err()))
+ return
+ default:
+ n, err := stream.Read(buf)
+ if err != nil {
+ slog.Error("read pod logs failed", slog.Any("error", err), slog.String("srv_name", srvName))
+ return
+ }
+ if n == 0 {
+ time.Sleep(5 * time.Second)
+ }
+
+ if n > 0 {
+ _, err = c.Writer.Write(buf[:n])
+ if err != nil {
+ slog.Error("write data failed", "error", err)
+ }
+ c.Writer.Flush()
+ slog.Info("send pod logs", slog.String("srv_name", srvName), slog.String("srv_name", srvName), slog.Int("len", n))
+ }
+ }
+
+ }
+}
+
+func (s *K8sHandler) ServiceStatusAll(c *gin.Context) {
+ allStatus := make(map[string]*types.StatusResponse)
+ for index := range s.clusterPool.Clusters {
+ cluster := s.clusterPool.Clusters[index]
+ services, err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).
+ List(c.Request.Context(), metav1.ListOptions{})
+ if err != nil {
+ slog.Error("get image status all failed, cannot get service infos", slog.Any("error", err))
+ //continue to next in multi cluster
+ continue
+ }
+
+ for _, srv := range services.Items {
+ deployIDStr := srv.Annotations[component.KeyDeployID]
+ deployID, _ := strconv.ParseInt(deployIDStr, 10, 64)
+ deployTypeStr := srv.Annotations[component.KeyDeployType]
+ deployType, err := strconv.ParseInt(deployTypeStr, 10, 64)
+ if err != nil {
+ deployType = 0
+ }
+ userID := srv.Annotations[component.KeyUserID]
+ deploySku := srv.Annotations[component.KeyDeploySKU]
+ orderDetailIdStr := srv.Annotations[component.KeyOrderDetailID]
+ orderDetailId, err := strconv.ParseInt(orderDetailIdStr, 10, 64)
+ if err != nil {
+ orderDetailId = 0
+ }
+ status := &types.StatusResponse{
+ DeployID: deployID,
+ UserID: userID,
+ DeployType: int(deployType),
+ ServiceName: srv.Name,
+ DeploySku: deploySku,
+ OrderDetailID: orderDetailId,
+ }
+ allStatus[srv.Name] = status
+ if srv.IsFailed() {
+ status.Code = common.DeployFailed
+ continue
+ }
+
+ if srv.IsReady() {
+ podNames, err := s.GetServicePods(c.Request.Context(), cluster, srv.Name, s.k8sNameSpace, 1)
+ if err != nil {
+ slog.Error("get image status failed, cannot get pods info", slog.Any("error", err))
+ status.Code = common.Running
+ continue
+ }
+ status.Replica = len(podNames)
+ if len(podNames) == 0 {
+ status.Code = common.Sleeping
+ continue
+ }
+
+ status.Code = common.Running
+ continue
+ }
+
+ // default to deploying
+ status.Code = common.Deploying
+ }
+ }
+
+ c.JSON(http.StatusOK, allStatus)
+}
+
+func (s *K8sHandler) GetServicePods(ctx context.Context, cluster cluster.Cluster, srvName string, namespace string, limit int64) ([]string, error) {
+ labelSelector := fmt.Sprintf("serving.knative.dev/service=%s", srvName)
+ // Get the list of Pods based on the label selector
+ opts := metav1.ListOptions{
+ LabelSelector: labelSelector,
+ }
+ if limit > 0 {
+ opts = metav1.ListOptions{
+ LabelSelector: labelSelector,
+ Limit: limit,
+ }
+ }
+ pods, err := cluster.Client.CoreV1().Pods(namespace).List(ctx, opts)
+ if err != nil {
+ return nil, err
+ }
+
+ // Extract the Pod names
+ var podNames []string
+ for _, pod := range pods.Items {
+ podNames = append(podNames, pod.Name)
+ }
+
+ return podNames, nil
+}
+
+func (s *K8sHandler) GetClusterInfo(c *gin.Context) {
+ clusterRes := []types.CluserResponse{}
+ for index := range s.clusterPool.Clusters {
+ cls := s.clusterPool.Clusters[index]
+ cInfo, err := s.clusterPool.ClusterStore.ByClusterConfig(c.Request.Context(), cls.ID)
+ if err != nil {
+ slog.Error("get cluster info failed", slog.Any("error", err))
+ continue
+ }
+ clusterInfo := types.CluserResponse{}
+ clusterInfo.Region = cInfo.Region
+ clusterInfo.Zone = cInfo.Zone
+ clusterInfo.Provider = cInfo.Provider
+ clusterInfo.ClusterID = cInfo.ClusterID
+ clusterInfo.ClusterName = fmt.Sprintf("cluster%d", index)
+ clusterRes = append(clusterRes, clusterInfo)
+
+ }
+ c.JSON(http.StatusOK, clusterRes)
+}
+
+func (s *K8sHandler) GetClusterInfoByID(c *gin.Context) {
+ clusterId := c.Params.ByName("id")
+ cInfo, _ := s.clusterPool.ClusterStore.ByClusterID(c.Request.Context(), clusterId)
+ clusterInfo := types.CluserResponse{}
+ clusterInfo.Region = cInfo.Region
+ clusterInfo.Zone = cInfo.Zone
+ clusterInfo.Provider = cInfo.Provider
+ clusterInfo.ClusterID = cInfo.ClusterID
+ clusterInfo.StorageClass = cInfo.StorageClass
+ client, err := s.clusterPool.GetClusterByID(c.Request.Context(), clusterId)
+ if err != nil {
+ slog.Error("fail to get cluster", slog.Any("error", err))
+ c.JSON(http.StatusNotFound, gin.H{"error": err.Error()})
+ return
+ }
+ nodes, err := cluster.GetNodeResources(client.Client, s.env)
+ if err == nil {
+ clusterInfo.Nodes = nodes
+ }
+
+ c.JSON(http.StatusOK, clusterInfo)
+}
+
+func (s *K8sHandler) getServiceNameFromRequest(c *gin.Context) string {
+ return c.Params.ByName("service")
+}
+
+func (s *K8sHandler) getPodNameFromRequest(c *gin.Context) string {
+ return c.Params.ByName("pod_name")
+}
+
+func (s *K8sHandler) GetServiceByName(c *gin.Context) {
+ var resp types.StatusResponse
+ var request = &types.CheckRequest{}
+ err := c.BindJSON(request)
+ if err != nil {
+ slog.Error("fail to parse input parameters", slog.Any("error", err))
+ resp.Code = -1
+ resp.Message = "fail to parse input parameters"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster config", slog.Any("error", err))
+ resp.Code = -1
+ resp.Message = "fail to get cluster config"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+ srvName := s.getServiceNameFromRequest(c)
+ srv, err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err != nil {
+ k8serr := new(k8serrors.StatusError)
+ if errors.As(err, &k8serr) {
+ if k8serr.Status().Code == http.StatusNotFound {
+ // service not exist
+ resp.Code = 0
+ resp.Message = "service not exist"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+ }
+ // get service with error
+ slog.Error("fail to get service with error", slog.Any("error", err))
+ resp.Code = -1
+ resp.Message = "fail to get service"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+
+ if srv == nil {
+ // service not exist
+ resp.Code = 0
+ resp.Message = "service not exist"
+ c.JSON(http.StatusOK, resp)
+ return
+ }
+
+ // service exist
+ deployIDStr := srv.Annotations[types.ResDeployID]
+ deployID, _ := strconv.ParseInt(deployIDStr, 10, 64)
+ resp.DeployID = deployID
+ resp.Code = 1
+ resp.Message = srvName
+ if srv.Status.URL != nil {
+ resp.Endpoint = srv.Status.URL.URL().String()
+ }
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) GetReplica(c *gin.Context) {
+ var resp types.ReplicaResponse
+ var request = &types.StatusRequest{}
+ err := c.BindJSON(request)
+ if err != nil {
+ slog.Error("fail to parse input parameters", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to parse input parameters"})
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster config", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to get cluster config"})
+ return
+ }
+ srvName := s.getServiceNameFromRequest(c)
+ srv, err := cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err != nil {
+ // get service with error
+ slog.Error("fail to get service", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to get service"})
+ return
+ }
+
+ if srv == nil {
+ // service not exist
+ slog.Error("service not exist")
+ c.JSON(http.StatusNotFound, gin.H{"error": "service not exist"})
+ return
+ }
+ // revisionName := srv.Status.LatestReadyRevisionName
+ revisionName := srv.Status.LatestCreatedRevisionName
+ if len(revisionName) < 1 {
+ slog.Error("fail to get latest created revision")
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to get latest created revision"})
+ return
+ }
+ revision, err := cluster.KnativeClient.ServingV1().Revisions(s.k8sNameSpace).Get(c.Request.Context(), revisionName, metav1.GetOptions{})
+ if err != nil {
+ slog.Error("fail to get revision with error", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to get revision with error"})
+ return
+ }
+
+ if revision == nil {
+ slog.Error("revision not exist")
+ c.JSON(http.StatusNotFound, gin.H{"error": "revision not exist"})
+ return
+ }
+ instList, err := s.s.GetServicePodsWithStatus(c.Request.Context(), *cluster, srvName, s.k8sNameSpace)
+ if err != nil {
+ slog.Error("fail to get service pod name list", slog.Any("error", err))
+ c.JSON(http.StatusNotFound, gin.H{"error": "fail to get service pod name list"})
+ return
+ }
+
+ // revision exist
+ resp.Code = 1
+ resp.Message = srvName
+ resp.ActualReplica = int(*revision.Status.ActualReplicas)
+ resp.DesiredReplica = int(*revision.Status.DesiredReplicas)
+ resp.Instances = instList
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) UpdateCluster(c *gin.Context) {
+ var resp types.UpdateClusterResponse
+ var request = &database.ClusterInfo{}
+ err := c.BindJSON(request)
+ if err != nil {
+ slog.Error("fail to parse input parameters", slog.Any("error", err))
+ resp.Code = -1
+ resp.Message = "fail to parse input parameters"
+ c.JSON(http.StatusBadRequest, resp)
+ return
+ }
+ err = s.clusterPool.ClusterStore.Update(c, *request)
+ if err != nil {
+ slog.Error("fail to update cluster", slog.Any("error", err))
+ resp.Code = -1
+ resp.Message = "fail to update cluster"
+ c.JSON(http.StatusInternalServerError, resp)
+ return
+ }
+ resp.Code = 0
+ resp.Message = "succeed to update cluster"
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) PurgeService(c *gin.Context) {
+ var resp types.PurgeResponse
+ var request = &types.PurgeRequest{}
+ err := c.BindJSON(request)
+ if err != nil {
+ slog.Error("fail to parse input parameters", slog.Any("error", err))
+ resp.Code = -1
+ resp.Message = "fail to parse cluster id"
+ c.JSON(http.StatusBadRequest, resp)
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster config", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to get cluster config"})
+ return
+ }
+ srvName := s.getServiceNameFromRequest(c)
+ _, err = cluster.KnativeClient.ServingV1().Services(s.k8sNameSpace).
+ Get(c.Request.Context(), srvName, metav1.GetOptions{})
+ if err != nil {
+ k8serr := new(k8serrors.StatusError)
+ if errors.As(err, &k8serr) {
+ if k8serr.Status().Code == http.StatusNotFound {
+ slog.Info("service not exist", slog.String("srv_name", srvName), slog.Any("k8s_err", k8serr))
+ }
+ }
+ slog.Error("purge service failed, cannot get service info", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ } else {
+ // 1 delete service
+ err = s.removeServiceForcely(c, cluster, srvName)
+ if err != nil {
+ slog.Error("failed to delete service ", slog.String("srv_name", srvName), slog.Any("error", err),
+ slog.String("srv_name", srvName))
+ resp.Code = -1
+ resp.Message = "failed to get service status"
+ c.JSON(http.StatusInternalServerError, resp)
+ return
+ }
+ }
+
+ // 2 clean up pvc
+ if cluster.StorageClass != "" && request.DeployType == types.FinetuneType {
+ err = cluster.Client.CoreV1().PersistentVolumeClaims(s.k8sNameSpace).Delete(c, srvName, metav1.DeleteOptions{})
+ if err != nil {
+ slog.Error("fail to delete pvc", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to delete pvc"})
+ return
+ }
+ slog.Info("persistent volume claims deleted.", slog.String("srv_name", srvName))
+ }
+ slog.Info("service deleted.", slog.String("srv_name", srvName))
+ resp.Code = 0
+ resp.Message = "succeed to clean up service"
+ c.JSON(http.StatusOK, resp)
+}
+
+func (s *K8sHandler) GetServiceInfo(c *gin.Context) {
+ var resp types.ServiceInfoResponse
+ var request = &types.ServiceRequest{}
+ err := c.BindJSON(request)
+ if err != nil {
+ slog.Error("fail to parse input parameters", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to parse input parameters"})
+ return
+ }
+ cluster, err := s.clusterPool.GetClusterByID(c, request.ClusterID)
+ if err != nil {
+ slog.Error("fail to get cluster config", slog.Any("error", err))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "fail to get cluster config"})
+ return
+ }
+
+ srvName := s.getServiceNameFromRequest(c)
+ podNames, err := s.GetServicePods(c.Request.Context(), *cluster, srvName, s.k8sNameSpace, -1)
+ if err != nil {
+ slog.Error("failed to read image logs, cannot get pods info", slog.Any("error", err), slog.String("srv_name", srvName))
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get pods info"})
+ return
+ }
+ resp.PodNames = podNames
+ resp.ServiceName = srvName
+ c.JSON(http.StatusOK, resp)
+}
+
+
+
package handler
+
+import (
+ "log/slog"
+ "net/http"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/builder/deploy/cluster"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/runner/component"
+)
+
+type ArgoHandler struct {
+ clusterPool *cluster.ClusterPool
+ workflowNameSpace string
+ modelDockerRegBase string
+ config *config.Config
+ wfc component.WorkFlowComponent
+}
+
+func NewArgoHandler(config *config.Config, clusterPool *cluster.ClusterPool) (*ArgoHandler, error) {
+ wfc := component.NewWorkFlowComponent(config, clusterPool)
+ return &ArgoHandler{
+ clusterPool: clusterPool,
+ config: config,
+ wfc: wfc,
+ modelDockerRegBase: config.Model.DockerRegBase,
+ workflowNameSpace: config.Argo.Namespace,
+ }, nil
+}
+
+// create workflow
+func (a *ArgoHandler) CreateWorkflow(ctx *gin.Context) {
+ var req types.ArgoWorkFlowReq
+ err := ctx.ShouldBindJSON(&req)
+ if err != nil {
+ slog.Error("bad order request format", "error", err)
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ wf, err := a.wfc.CreateWorkflow(ctx, req)
+ if err != nil {
+ slog.Error("fail to create workflow", slog.Any("error", err), slog.Any("req", req))
+ ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+ res := types.ArgoWorkFlowRes{
+ ID: wf.ID,
+ Username: wf.Username,
+ TaskName: wf.TaskName,
+ TaskId: wf.TaskId,
+ TaskType: wf.TaskType,
+ TaskDesc: wf.TaskDesc,
+ RepoIds: wf.RepoIds,
+ RepoType: wf.RepoType,
+ SubmitTime: wf.SubmitTime,
+ }
+
+ ctx.JSON(http.StatusOK, res)
+}
+
+// list workflows
+func (a *ArgoHandler) ListWorkflows(ctx *gin.Context) {
+ username := ctx.Query("username")
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format of page and per", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ wfs, total, err := a.wfc.FindWorkFlows(ctx, username, per, page)
+ if err != nil {
+ slog.Error("fail to list workflows", slog.Any("error", err))
+ ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+ ctx.JSON(http.StatusOK, gin.H{
+ "total": total,
+ "list": wfs,
+ })
+}
+
+func (a *ArgoHandler) DeleteWorkflow(ctx *gin.Context) {
+ id := ctx.Param("id")
+ var req = &types.ArgoWorkFlowDeleteReq{}
+ err := ctx.BindJSON(req)
+ if err != nil {
+ slog.Error("bad request format", "error", err)
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ idInt64, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("fail to convert id to int64", slog.Any("error", err), slog.Any("id", id))
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ err = a.wfc.DeleteWorkflow(ctx, idInt64, req.Username)
+ if err != nil {
+ slog.Error("fail to delete workflow", slog.Any("error", err))
+ ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+ slog.Info("Deleted argo workflow successfully", slog.String("id", id))
+ httpbase.OK(ctx, nil)
+}
+
+func (a *ArgoHandler) GetWorkflow(ctx *gin.Context) {
+ id := ctx.Param("id")
+ var req = &types.ArgoWorkFlowGetReq{}
+ err := ctx.BindJSON(req)
+ if err != nil {
+ slog.Error("bad request format", "error", err)
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ idInt64, err := strconv.ParseInt(id, 10, 64)
+ if err != nil {
+ slog.Error("fail to convert id to int64", slog.Any("error", err), slog.Any("id", id))
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+ return
+ }
+ wf, err := a.wfc.GetWorkflow(ctx, idInt64, req.Username)
+ if err != nil {
+ slog.Error("fail to get workflow", slog.Any("error", err))
+ ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+ return
+ }
+ res := types.ArgoWorkFlowRes{
+ ID: wf.ID,
+ Username: wf.Username,
+ TaskName: wf.TaskName,
+ Image: wf.Image,
+ TaskId: wf.TaskId,
+ TaskType: wf.TaskType,
+ TaskDesc: wf.TaskDesc,
+ RepoIds: wf.RepoIds,
+ RepoType: wf.RepoType,
+ SubmitTime: wf.SubmitTime,
+ StartTime: wf.StartTime,
+ EndTime: wf.EndTime,
+ Datasets: wf.Datasets,
+ ResultURL: wf.ResultURL,
+ DownloadURL: wf.DownloadURL,
+ FailuresURL: wf.FailuresURL,
+ Status: wf.Status,
+ }
+ ctx.JSON(http.StatusOK, res)
+}
+
+
+
package router
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/builder/deploy/cluster"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/runner/handler"
+)
+
+func NewHttpServer(config *config.Config) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+
+ clusterPool, err := cluster.NewClusterPool()
+ if err != nil {
+ slog.Error("falied to build kubeconfig", "error", err)
+ return nil, fmt.Errorf("failed to build kubeconfig,%w", err)
+ }
+
+ k8sHandler, err := handler.NewK8sHandler(config, clusterPool)
+ if err != nil {
+ return nil, fmt.Errorf("failed to build NewHttpServer,%w", err)
+ }
+ apiGroup := r.Group("/api/v1")
+ service := apiGroup.Group("/service")
+ {
+ service.POST("/:service/run", k8sHandler.RunService)
+ service.PUT("/:service/update", k8sHandler.UpdateService)
+ service.POST("/:service/stop", k8sHandler.StopService)
+ service.GET("/:service/status", k8sHandler.ServiceStatus)
+ service.GET("/:service/logs", k8sHandler.ServiceLogs)
+ service.GET("/:service/logs/:pod_name", k8sHandler.ServiceLogsByPod)
+ service.GET("/:service/info", k8sHandler.GetServiceInfo)
+ service.GET("/status-all", k8sHandler.ServiceStatusAll)
+ service.GET("/:service/get", k8sHandler.GetServiceByName)
+ service.GET("/:service/replica", k8sHandler.GetReplica)
+ service.DELETE("/:service/purge", k8sHandler.PurgeService)
+
+ }
+ cluster := apiGroup.Group("/cluster")
+ {
+ cluster.GET("", k8sHandler.GetClusterInfo)
+ cluster.GET("/:id", k8sHandler.GetClusterInfoByID)
+ cluster.PUT("/:id", k8sHandler.UpdateCluster)
+ }
+ argoHandler, err := handler.NewArgoHandler(config, clusterPool)
+ if err != nil {
+ return nil, fmt.Errorf("failed to build NewHttpServer,%w", err)
+ }
+
+ workflows := apiGroup.Group("/workflows")
+ {
+ workflows.POST("", argoHandler.CreateWorkflow)
+ workflows.GET("", argoHandler.ListWorkflows)
+ workflows.DELETE("/:id", argoHandler.DeleteWorkflow)
+ workflows.GET("/:id", argoHandler.GetWorkflow)
+ }
+
+ return r, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/accounting"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+var ErrUserNotFound = errors.New("user not found, please login first")
+
+type AccessTokenComponent interface {
+ Create(ctx context.Context, req *types.CreateUserTokenRequest) (*database.AccessToken, error)
+ Delete(ctx context.Context, req *types.DeleteUserTokenRequest) error
+ Check(ctx context.Context, req *types.CheckAccessTokenReq) (types.CheckAccessTokenResp, error)
+ GetTokens(ctx context.Context, username, app string) ([]types.CheckAccessTokenResp, error)
+ RefreshToken(ctx context.Context, userName, tokenName, app string, newExpiredAt time.Time) (types.CheckAccessTokenResp, error)
+ GetOrCreateFirstAvaiToken(ctx context.Context, userName, app, tokenName string) (string, error)
+}
+
+func NewAccessTokenComponent(config *config.Config) (AccessTokenComponent, error) {
+ var err error
+ ac, err := accounting.NewAccountingClient(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create accounting clent,error:%w", err)
+ }
+ c := &accessTokenComponentImpl{}
+ c.ts = database.NewAccessTokenStore()
+ c.us = database.NewUserStore()
+ c.gs, err = git.NewGitServer(config)
+ c.acctClient = ac
+ c.config = config
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ return c, nil
+}
+
+type accessTokenComponentImpl struct {
+ ts database.AccessTokenStore
+ us database.UserStore
+ gs gitserver.GitServer
+ acctClient accounting.AccountingClient
+ config *config.Config
+}
+
+func (c *accessTokenComponentImpl) Create(ctx context.Context, req *types.CreateUserTokenRequest) (*database.AccessToken, error) {
+ user, err := c.us.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("fail to find user,error:%w", err)
+ }
+
+ exist, err := c.ts.IsExist(ctx, req.Username, req.TokenName, string(req.Application))
+ if err != nil {
+ return nil, fmt.Errorf("fail to check if token exists,error:%w", err)
+ }
+ if exist {
+ return nil, fmt.Errorf("token name duplicated, token_name:%s, app:%s", req.TokenName, req.Application)
+ }
+
+ var token *database.AccessToken
+ // csghub token is shared with git server
+ if req.Application == types.AccessTokenAppGit {
+ if c.gs != nil {
+ token, err = c.gs.CreateUserToken(req)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create git user access token,error:%w", err)
+ }
+ } else {
+ tokenContent := c.genUnique()
+ token = &database.AccessToken{
+ Name: req.TokenName,
+ Token: tokenContent,
+ UserID: user.ID,
+ Application: req.Application,
+ Permission: req.Permission,
+ IsActive: true,
+ }
+ }
+ token.UserID = user.ID
+ token.Application = req.Application
+ } else {
+ tokenValue := c.genUnique()
+ token = &database.AccessToken{
+ Name: req.TokenName,
+ Token: tokenValue,
+ UserID: user.ID,
+ Application: req.Application,
+ Permission: req.Permission,
+ IsActive: true,
+ }
+ }
+
+ if req.ExpiredAt.After(time.Now()) {
+ token.ExpiredAt = req.ExpiredAt
+ }
+
+ err = c.createUserToken(ctx, token, user)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create database user access token,error:%w", err)
+ }
+
+ if req.Application == types.AccessTokenAppMirror {
+ quota, err := c.acctClient.GetQuotaByID(req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get quota by username,error:%w", err)
+ }
+ if quota == nil {
+ _, err := c.acctClient.CreateOrUpdateQuota(req.Username, types.ACCT_QUOTA_REQ{
+ RepoCountLimit: c.config.MultiSync.DefaultRepoCountLimit,
+ SpeedLimit: c.config.MultiSync.DefaultSpeedLimit,
+ TrafficLimit: c.config.MultiSync.DefaultTrafficLimit,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("fail to create quota for new mirror token,error:%w", err)
+ }
+ }
+ }
+
+ return token, nil
+}
+
+func (c *accessTokenComponentImpl) genUnique() string {
+ // TODO:change
+ return strings.ReplaceAll(uuid.NewString(), "-", "")
+}
+
+func (c *accessTokenComponentImpl) Delete(ctx context.Context, req *types.DeleteUserTokenRequest) error {
+ ue, err := c.us.IsExist(ctx, req.Username)
+ if !ue {
+ return fmt.Errorf("user does not exists,error:%w", err)
+ }
+ te, err := c.ts.IsExist(ctx, req.Username, req.TokenName, string(req.Application))
+ if !te {
+ return fmt.Errorf("user access token does not exists,error:%w", err)
+ }
+
+ if req.Application == types.AccessTokenAppGit {
+ err = c.gs.DeleteUserToken(req)
+ if err != nil {
+ return fmt.Errorf("failed to delete git user access token,error:%w", err)
+ }
+ }
+
+ err = c.ts.Delete(ctx, req.Username, req.TokenName, string(req.Application))
+ if err != nil {
+ return fmt.Errorf("failed to delete database user access token,error,error:%w", err)
+ }
+ return nil
+}
+
+func (c *accessTokenComponentImpl) Check(ctx context.Context, req *types.CheckAccessTokenReq) (types.CheckAccessTokenResp, error) {
+ var resp types.CheckAccessTokenResp
+ t, err := c.ts.FindByToken(ctx, req.Token, req.Application)
+ if err != nil {
+ return resp, fmt.Errorf("failed to find database user access token,error:%w", err)
+ }
+
+ resp.Token = t.Token
+ resp.TokenName = t.Name
+ resp.Application = t.Application
+ resp.Permission = t.Permission
+ resp.Username = t.User.Username
+ resp.UserUUID = t.User.UUID
+ resp.ExpireAt = t.ExpiredAt
+ return resp, nil
+}
+
+func (c *accessTokenComponentImpl) GetTokens(ctx context.Context, username, app string) ([]types.CheckAccessTokenResp, error) {
+ var resps []types.CheckAccessTokenResp
+ tokens, err := c.ts.FindByUser(ctx, username, app)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return nil, fmt.Errorf("failed to find database user access token,error:%w", err)
+ }
+
+ for _, t := range tokens {
+ var resp types.CheckAccessTokenResp
+ resp.Token = t.Token
+ resp.TokenName = t.Name
+ resp.Application = t.Application
+ resp.Permission = t.Permission
+ resp.Username = t.User.Username
+ resp.UserUUID = t.User.UUID
+ resp.ExpireAt = t.ExpiredAt
+
+ resps = append(resps, resp)
+ }
+ return resps, nil
+}
+
+func (c *accessTokenComponentImpl) RefreshToken(ctx context.Context, userName, tokenName, app string, newExpiredAt time.Time) (types.CheckAccessTokenResp, error) {
+ var resp types.CheckAccessTokenResp
+ t, err := c.ts.FindByTokenName(ctx, userName, tokenName, app)
+ if err != nil {
+ return types.CheckAccessTokenResp{}, fmt.Errorf("failed to find database user access token,error:%w", err)
+ }
+
+ var newTokenValue string
+ req := &types.CreateUserTokenRequest{
+ Username: userName,
+ TokenName: t.Name,
+ Application: t.Application,
+ Permission: t.Permission,
+ }
+ // csghub token is shared with git server
+ if req.Application == "" || req.Application == types.AccessTokenAppCSGHub {
+ // TODO:allow git client to refresh token
+ // git server cannot create tokens with the same nanme
+ err := c.gs.DeleteUserToken(&types.DeleteUserTokenRequest{
+ Username: userName,
+ TokenName: t.Name,
+ })
+ if err != nil {
+ return resp, fmt.Errorf("fail to delete old git user access token,error:%w", err)
+ }
+ newToken, err := c.gs.CreateUserToken(req)
+ if err != nil {
+ return resp, fmt.Errorf("fail to create git user access token,error:%w", err)
+ }
+ newTokenValue = newToken.Token
+ } else {
+ newTokenValue = c.genUnique()
+ }
+
+ newToken, err := c.ts.Refresh(ctx, t, newTokenValue, newExpiredAt)
+ if err != nil {
+ return resp, fmt.Errorf("fail to refresh access token with new token value,error:%w", err)
+ }
+
+ resp.Token = newToken.Token
+ resp.TokenName = newToken.Name
+ resp.Application = newToken.Application
+ resp.Permission = newToken.Permission
+ resp.Username = newToken.User.Username
+ resp.UserUUID = newToken.User.UUID
+ resp.ExpireAt = newToken.ExpiredAt
+
+ return resp, nil
+}
+
+func (c *accessTokenComponentImpl) GetOrCreateFirstAvaiToken(ctx context.Context, userName, app, tokenName string) (string, error) {
+ tokens, err := c.GetTokens(ctx, userName, app)
+ if err != nil {
+ return "", fmt.Errorf("failed to select user %s access %s tokens, error:%w", userName, app, err)
+ }
+ if len(tokens) > 0 {
+ return tokens[0].Token, nil
+ }
+
+ req := types.CreateUserTokenRequest{
+ Username: userName,
+ TokenName: tokenName,
+ Application: types.AccessTokenApp(app),
+ Permission: "",
+ }
+
+ token, err := c.Create(ctx, &req)
+ if err != nil {
+ return "", fmt.Errorf("failed to create user %s access %s token, error:%w", userName, app, err)
+ }
+
+ return token.Token, nil
+}
+
+func (c *accessTokenComponentImpl) createUserToken(ctx context.Context, newToken *database.AccessToken, user database.User) error {
+ err := c.ts.Create(ctx, newToken)
+ if err != nil {
+ return fmt.Errorf("fail to create user %s new %s token, error: %w", user.Username, newToken.Application, err)
+ }
+
+ if newToken.Application == types.AccessTokenAppStarship {
+ // charge 100 credit for create starship token by call accounting service
+ err = c.presentForNewAccessToken(user)
+ if err != nil {
+ slog.Error("fail to charge for new starship user with retry 3 times", slog.Any("user.uuid", user.UUID), slog.Any("err", err))
+ }
+ }
+
+ return nil
+}
+
+func (c *accessTokenComponentImpl) presentForNewAccessToken(user database.User) error {
+ var err error
+ req := types.ACTIVITY_REQ{
+ ID: types.StarShipNewUser.ID,
+ Value: types.StarShipNewUser.Value,
+ OpUID: user.Username,
+ OpDesc: types.StarShipNewUser.OpDesc,
+ }
+ // retry 3 time
+ for i := 0; i < 3; i++ {
+ _, err = c.acctClient.PresentAccountingUser(user.UUID, req)
+ if err == nil {
+ return nil
+ }
+ }
+ return err
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/golang-jwt/jwt/v5"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type jwtComponentImpl struct {
+ SigningKey []byte
+ ValidTime time.Duration
+ us database.UserStore
+}
+
+type JwtComponent interface {
+ // GenerateToken generate a jwt token, and return the token and signed string
+ GenerateToken(ctx context.Context, req types.CreateJWTReq) (claims *types.JWTClaims, signed string, err error)
+ ParseToken(ctx context.Context, token string) (user *types.User, err error)
+}
+
+func NewJwtComponent(signKey string, validHour int) JwtComponent {
+ return &jwtComponentImpl{
+ SigningKey: []byte(signKey),
+ ValidTime: time.Duration(validHour) * time.Hour,
+ us: database.NewUserStore(),
+ }
+}
+
+// GenerateToken generate a jwt token, and return the token and signed string
+func (c *jwtComponentImpl) GenerateToken(ctx context.Context, req types.CreateJWTReq) (claims *types.JWTClaims, signed string, err error) {
+ u, err := c.us.FindByUUID(ctx, req.UUID)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to find user by uuid '%s',error: %w", req.UUID, err)
+ }
+ expireAt := jwt.NewNumericDate(time.Now().Add(c.ValidTime))
+ claims = &types.JWTClaims{
+ UUID: u.UUID,
+ CurrentUser: u.Username,
+ RegisteredClaims: jwt.RegisteredClaims{
+ ExpiresAt: expireAt,
+ Issuer: "OpenCSG",
+ },
+ }
+
+ token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+ signed, err = token.SignedString(c.SigningKey)
+ if err != nil {
+ return nil, "", fmt.Errorf("generate jwt token failed: %w", err)
+ }
+
+ return claims, signed, nil
+}
+
+func (c *jwtComponentImpl) ParseToken(ctx context.Context, token string) (user *types.User, err error) {
+ claims := &types.JWTClaims{}
+ _, err = jwt.ParseWithClaims(token, claims,
+ func(token *jwt.Token) (interface{}, error) {
+ return c.SigningKey, nil
+ },
+ jwt.WithIssuer("OpenCSG"),
+ )
+
+ if err != nil {
+ return nil, fmt.Errorf("parse jwt token failed: %w", err)
+ }
+
+ dbu, err := c.us.FindByUsername(ctx, claims.CurrentUser)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user by name '%s', %w", claims.CurrentUser, err)
+ }
+
+ // create new user object
+ u := &types.User{
+ UUID: dbu.UUID,
+ Username: dbu.Username,
+ Email: dbu.Email,
+ Roles: dbu.Roles(),
+ CanChangeUserName: dbu.CanChangeUserName,
+ }
+ return u, nil
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/git/membership"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type memberComponentImpl struct {
+ memberStore database.MemberStore
+ orgStore database.OrgStore
+ userStore database.UserStore
+ gitServer gitserver.GitServer
+ gitMemberShip membership.GitMemerShip
+ config *config.Config
+}
+
+type MemberComponent interface {
+ OrgMembers(ctx context.Context, orgName, currentUser string, pageSize, page int) ([]types.Member, int, error)
+ InitRoles(ctx context.Context, org *database.Organization) error
+ SetAdmin(ctx context.Context, org *database.Organization, user *database.User) error
+ ChangeMemberRole(ctx context.Context, orgName, userName, operatorName, oldRole, newRole string) error
+ GetMemberRole(ctx context.Context, orgName, userName string) (membership.Role, error)
+ AddMembers(ctx context.Context, orgName string, users []string, operatorName string, role string) error
+ AddMember(ctx context.Context, orgName, userName, operatorName string, role string) error
+ Delete(ctx context.Context, orgName, userName, operatorName string, role string) error
+}
+
+func NewMemberComponent(config *config.Config) (MemberComponent, error) {
+ var gms membership.GitMemerShip
+ gs, err := git.NewGitServer(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git server:%w", err)
+ }
+ if config.GitServer.Type == types.GitServerTypeGitea {
+ gms, err = git.NewMemberShip(*config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create git membership:%w", err)
+ }
+ }
+ return &memberComponentImpl{
+ memberStore: database.NewMemberStore(),
+ orgStore: database.NewOrgStore(),
+ userStore: database.NewUserStore(),
+ gitServer: gs,
+ gitMemberShip: gms,
+ config: config,
+ }, nil
+}
+
+func (c *memberComponentImpl) OrgMembers(ctx context.Context, orgName, currentUser string, pageSize, page int) ([]types.Member, int, error) {
+ var (
+ org database.Organization
+ user database.User
+ err error
+ )
+ org, err = c.orgStore.FindByPath(ctx, orgName)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to find org,caused by:%w", err)
+ }
+ moreDetail := false
+ user, err = c.userStore.FindByUsername(ctx, currentUser)
+ if err == nil && user.ID > 0 {
+ m, err := c.memberStore.Find(ctx, org.ID, user.ID)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ slog.Error("failed to find member", "error", err)
+ }
+ //if current user belongs to org, show more detail member info
+ if m != nil {
+ moreDetail = true
+ }
+ }
+
+ dbmembers, total, err := c.memberStore.OrganizationMembers(ctx, org.ID, pageSize, page)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to find org members,caused by:%w", err)
+ }
+ var members []types.Member
+ for _, dbmember := range dbmembers {
+ m := types.Member{
+ UUID: dbmember.User.UUID,
+ Avatar: dbmember.User.Avatar,
+ Username: dbmember.User.Username,
+ Nickname: dbmember.User.NickName,
+ }
+ if moreDetail {
+ m.Role = dbmember.Role
+ m.LastLoginAt = dbmember.User.LastLoginAt
+ }
+ members = append(members, m)
+ }
+ return members, total, nil
+}
+
+func (c *memberComponentImpl) InitRoles(ctx context.Context, org *database.Organization) error {
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ return c.gitMemberShip.AddRoles(ctx, org.Name,
+ []membership.Role{membership.RoleAdmin, membership.RoleRead, membership.RoleWrite})
+ } else {
+ return nil
+ }
+}
+
+func (c *memberComponentImpl) SetAdmin(ctx context.Context, org *database.Organization, user *database.User) error {
+ var (
+ err error
+ )
+ err = c.memberStore.Add(ctx, org.ID, user.ID, string(membership.RoleAdmin))
+ if err != nil {
+ err = fmt.Errorf("failed to create member,caused by:%w", err)
+ return err
+ }
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ return c.gitMemberShip.AddMember(ctx, org.Name, user.Username, membership.RoleAdmin)
+ } else {
+ return nil
+ }
+}
+
+func (c *memberComponentImpl) ChangeMemberRole(ctx context.Context, orgName, userName, operatorName, oldRole, newRole string) error {
+ err := c.Delete(ctx, orgName, userName, operatorName, oldRole)
+ if err != nil {
+ return fmt.Errorf("failed to delete old role,error:%w", err)
+ }
+ err = c.AddMember(ctx, orgName, userName, operatorName, newRole)
+ if err != nil {
+ return fmt.Errorf("failed to create new role,error:%w", err)
+ }
+
+ return nil
+}
+
+func (c *memberComponentImpl) GetMemberRole(ctx context.Context, orgName, userName string) (membership.Role, error) {
+ var (
+ org database.Organization
+ user database.User
+ err error
+ )
+ org, err = c.orgStore.FindByPath(ctx, orgName)
+ if err != nil {
+ return membership.RoleUnknown, fmt.Errorf("failed to find org,caused by:%w", err)
+ }
+ user, err = c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return membership.RoleUnknown, fmt.Errorf("failed to find user,caused by:%w", err)
+ }
+ m, err := c.memberStore.Find(ctx, org.ID, user.ID)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return membership.RoleUnknown, fmt.Errorf("failed to check memberhsip existence,caused by:%w", err)
+ }
+ if m == nil {
+ return membership.RoleUnknown, nil
+ }
+ return c.toGitRole(m.Role), nil
+}
+
+func (c *memberComponentImpl) AddMembers(ctx context.Context, orgName string, users []string, operatorName string, role string) error {
+ var (
+ org database.Organization
+ op database.User
+ user database.User
+ err error
+ )
+ org, err = c.orgStore.FindByPath(ctx, orgName)
+ if err != nil {
+ return fmt.Errorf("failed to find org,org:%s,caused by:%w", orgName, err)
+ }
+ op, err = c.userStore.FindByUsername(ctx, operatorName)
+ if err != nil {
+ return fmt.Errorf("failed to find op user,user:%s,caused by:%w", operatorName, err)
+ }
+ opMember, err := c.memberStore.Find(ctx, org.ID, op.ID)
+ if err != nil {
+ return fmt.Errorf("failed to get op user membership,user:%s,caused by:%w", operatorName, err)
+ }
+ if !c.allowAddMember(opMember) {
+ return fmt.Errorf("add member operation not allowed, user:%s", operatorName)
+ }
+
+ for _, userName := range users {
+ user, err = c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return fmt.Errorf("failed to find user, user:%s,caused by:%w", userName, err)
+ }
+ m, err := c.memberStore.Find(ctx, org.ID, user.ID)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("failed to check memberhsip existence, user:%s,caused by:%w", userName, err)
+ }
+ //skip existing member
+ if m != nil {
+ continue
+ }
+ err = c.memberStore.Add(ctx, org.ID, user.ID, role)
+ if err != nil {
+ err = fmt.Errorf("failed to create db member, org:%s, user:%s,caused by:%w", orgName, userName, err)
+ return err
+ }
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ err = c.gitMemberShip.AddMember(ctx, orgName, userName, c.toGitRole(role))
+ if err != nil {
+ return fmt.Errorf("failed to add git member, org:%s, user:%s caused by:%w", orgName, userName, err)
+ }
+ }
+ }
+
+ return nil
+}
+
+func (c *memberComponentImpl) AddMember(ctx context.Context, orgName, userName, operatorName string, role string) error {
+ return c.AddMembers(ctx, orgName, []string{userName}, operatorName, role)
+}
+
+func (c *memberComponentImpl) Delete(ctx context.Context, orgName, userName, operatorName string, role string) error {
+ var (
+ org database.Organization
+ op database.User
+ user database.User
+ err error
+ )
+ org, err = c.orgStore.FindByPath(ctx, orgName)
+ if err != nil {
+ return fmt.Errorf("failed to find org,caused by:%w", err)
+ }
+ op, err = c.userStore.FindByUsername(ctx, operatorName)
+ if err != nil {
+ return fmt.Errorf("failed to find user,caused by:%w", err)
+ }
+ opMember, err := c.memberStore.Find(ctx, org.ID, op.ID)
+ if err != nil {
+ return fmt.Errorf("failed to get op user membership,caused by:%w", err)
+ }
+ if !c.allowAddMember(opMember) {
+ return errors.New("operation not allowed")
+ }
+ user, err = c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return fmt.Errorf("failed to find user,caused by:%w", err)
+ }
+ m, err := c.memberStore.Find(ctx, org.ID, user.ID)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("failed to check memberhsip existence,caused by:%w", err)
+ }
+ //skip if not a member
+ if m == nil {
+ return nil
+ }
+ err = c.memberStore.Delete(ctx, org.ID, user.ID, role)
+ if err != nil {
+ err = fmt.Errorf("failed to delete member,caused by:%w", err)
+ return err
+ }
+ if c.config.GitServer.Type == types.GitServerTypeGitea {
+ return c.gitMemberShip.RemoveMember(ctx, orgName, userName, c.toGitRole(role))
+ } else {
+ return nil
+ }
+}
+
+func (c *memberComponentImpl) allowAddMember(u *database.Member) bool {
+ //TODO: check more roles
+ return u != nil && u.Role == string(membership.RoleAdmin)
+}
+
+func (c *memberComponentImpl) toGitRole(role string) membership.Role {
+ switch role {
+ case "admin":
+ return membership.RoleAdmin
+ case "write":
+ return membership.RoleWrite
+ case "read":
+ return membership.RoleRead
+ default:
+ return membership.RoleUnknown
+ }
+}
+
+
+
package component
+
+import (
+ "context"
+ "fmt"
+
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type namespaceComponentImpl struct {
+ ns database.NamespaceStore
+ os database.OrgStore
+}
+
+type NamespaceComponent interface {
+ GetInfo(ctx context.Context, path string) (*types.Namespace, error)
+}
+
+func NewNamespaceComponent(config *config.Config) (NamespaceComponent, error) {
+ return &namespaceComponentImpl{
+ ns: database.NewNamespaceStore(),
+ os: database.NewOrgStore(),
+ }, nil
+}
+
+func (c *namespaceComponentImpl) GetInfo(ctx context.Context, path string) (*types.Namespace, error) {
+ dbns, err := c.ns.FindByPath(ctx, path)
+ ns := &types.Namespace{
+ Path: dbns.Path,
+ Type: string(dbns.NamespaceType),
+ }
+ if dbns.NamespaceType == database.UserNamespace {
+ ns.Avatar = dbns.User.Avatar
+ } else if dbns.NamespaceType == database.OrgNamespace {
+ org, err := c.os.FindByPath(ctx, dbns.Path)
+ if err != nil {
+ return nil, fmt.Errorf("fail to get org info, path: %s, error: %w", path, err)
+ }
+ //overwrite namespace type with org type
+ ns.Type = string(org.OrgType)
+ ns.Avatar = org.Logo
+ } else {
+ return nil, fmt.Errorf("invalid namespace type: %s", dbns.NamespaceType)
+ }
+ return ns, err
+
+}
+
+
+
package component
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+type OrganizationComponent interface {
+ FixOrgData(ctx context.Context, org *database.Organization) (*database.Organization, error)
+ Create(ctx context.Context, req *types.CreateOrgReq) (*types.Organization, error)
+ Index(ctx context.Context, username, search string, per, page int) ([]types.Organization, int, error)
+ Get(ctx context.Context, orgName string) (*types.Organization, error)
+ Delete(ctx context.Context, req *types.DeleteOrgReq) error
+ Update(ctx context.Context, req *types.EditOrgReq) (*database.Organization, error)
+}
+
+func NewOrganizationComponent(config *config.Config) (OrganizationComponent, error) {
+ c := &organizationComponentImpl{}
+ c.orgStore = database.NewOrgStore()
+ c.nsStore = database.NewNamespaceStore()
+ c.userStore = database.NewUserStore()
+ var err error
+ c.gs, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create git server,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ c.msc, err = NewMemberComponent(config)
+ if err != nil {
+ newError := fmt.Errorf("fail to create membership component,error:%w", err)
+ slog.Error(newError.Error())
+ return nil, newError
+ }
+ return c, nil
+}
+
+type organizationComponentImpl struct {
+ orgStore database.OrgStore
+ nsStore database.NamespaceStore
+ userStore database.UserStore
+ gs gitserver.GitServer
+
+ msc MemberComponent
+}
+
+func (c *organizationComponentImpl) FixOrgData(ctx context.Context, org *database.Organization) (*database.Organization, error) {
+ user := org.User
+ req := new(types.CreateOrgReq)
+ req.Name = org.Name
+ req.Nickname = org.Nickname
+ req.Username = org.User.Username
+ req.Description = org.Description
+ err := c.gs.FixOrganization(req, *user)
+ if err != nil {
+ slog.Error("fix git org data has error", slog.Any("error", err))
+ }
+ // need to create roles for a new org before adding members
+ err = c.msc.InitRoles(ctx, org)
+ if err != nil {
+ slog.Error("fix organization role has error", slog.String("error", err.Error()))
+ }
+ // org creator defaults to be admin role
+ err = c.msc.SetAdmin(ctx, org, user)
+ return org, err
+}
+
+func (c *organizationComponentImpl) Create(ctx context.Context, req *types.CreateOrgReq) (*types.Organization, error) {
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user, error: %w", err)
+ }
+
+ es, err := c.nsStore.Exists(ctx, req.Name)
+ if err != nil {
+ return nil, err
+ }
+ if es {
+ return nil, errors.New("the name already exists")
+ }
+
+ dbOrg, err := c.gs.CreateOrganization(req, user)
+ if err != nil {
+ return nil, fmt.Errorf("failed create git organization, error: %w", err)
+ }
+ dbOrg.Homepage = req.Homepage
+ dbOrg.Logo = req.Logo
+ dbOrg.OrgType = req.OrgType
+ dbOrg.Verified = req.Verified
+ namespace := &database.Namespace{
+ Path: dbOrg.Name,
+ UserID: user.ID,
+ }
+ err = c.orgStore.Create(ctx, dbOrg, namespace)
+ if err != nil {
+ return nil, fmt.Errorf("failed create database organization, error: %w", err)
+ }
+ // need to create roles for a new org before adding members
+ err = c.msc.InitRoles(ctx, dbOrg)
+ if err != nil {
+ return nil, fmt.Errorf("failed init roles for organization, error: %w", err)
+ }
+ // org creator defaults to be admin role
+ err = c.msc.SetAdmin(ctx, dbOrg, &user)
+ if err != nil {
+ return nil, fmt.Errorf("failed set admin role for organization, error: %w", err)
+ }
+
+ org := &types.Organization{
+ Name: dbOrg.Name,
+ Nickname: dbOrg.Nickname,
+ Homepage: dbOrg.Homepage,
+ Logo: dbOrg.Logo,
+ OrgType: dbOrg.OrgType,
+ Verified: dbOrg.Verified,
+ }
+ return org, err
+}
+
+func (c *organizationComponentImpl) Index(ctx context.Context, username, search string, per, page int) ([]types.Organization, int, error) {
+ var (
+ err error
+ total int
+ u database.User
+ dborgs []database.Organization
+ )
+ u, err = c.userStore.FindByUsername(ctx, username)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to find user, error: %w", err)
+ }
+ if u.CanAdmin() {
+ dborgs, total, err = c.orgStore.Search(ctx, search, per, page)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get organizations for admin user, error: %w", err)
+ }
+ } else {
+ dborgs, total, err = c.orgStore.GetUserOwnOrgs(ctx, username)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to get organizations for owner, error: %w", err)
+ }
+ }
+ var orgs []types.Organization
+ for _, dborg := range dborgs {
+ org := types.Organization{
+ Name: dborg.Name,
+ Nickname: dborg.Nickname,
+ Homepage: dborg.Homepage,
+ Logo: dborg.Logo,
+ OrgType: dborg.OrgType,
+ Verified: dborg.Verified,
+ }
+ orgs = append(orgs, org)
+ }
+ return orgs, total, nil
+}
+
+func (c *organizationComponentImpl) Get(ctx context.Context, orgName string) (*types.Organization, error) {
+ dborg, err := c.orgStore.FindByPath(ctx, orgName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get organizations by name, error: %w", err)
+ }
+ org := &types.Organization{
+ Name: dborg.Name,
+ Nickname: dborg.Nickname,
+ Homepage: dborg.Homepage,
+ Logo: dborg.Logo,
+ OrgType: dborg.OrgType,
+ Verified: dborg.Verified,
+ }
+ return org, nil
+}
+
+func (c *organizationComponentImpl) Delete(ctx context.Context, req *types.DeleteOrgReq) error {
+ r, err := c.msc.GetMemberRole(ctx, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Name), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ if !r.CanAdmin() {
+ return fmt.Errorf("current user does not have permission to edit the organization, current user: %s", req.CurrentUser)
+ }
+ err = c.gs.DeleteOrganization(req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to delete git organizations, error: %w", err)
+ }
+ err = c.orgStore.Delete(ctx, req.Name)
+ if err != nil {
+ return fmt.Errorf("failed to delete database organizations, error: %w", err)
+ }
+ return nil
+}
+
+func (c *organizationComponentImpl) Update(ctx context.Context, req *types.EditOrgReq) (*database.Organization, error) {
+ r, err := c.msc.GetMemberRole(ctx, req.Name, req.CurrentUser)
+ if err != nil {
+ slog.Error("faild to get member role",
+ slog.String("org", req.Name), slog.String("user", req.CurrentUser),
+ slog.String("error", err.Error()))
+ }
+ if !r.CanAdmin() {
+ return nil, fmt.Errorf("current user does not have permission to edit the organization, current user: %s", req.CurrentUser)
+ }
+ org, err := c.orgStore.FindByPath(ctx, req.Name)
+ if err != nil {
+ return nil, fmt.Errorf("organization does not exists, error: %w", err)
+ }
+
+ if req.Nickname != nil {
+ org.Nickname = *req.Nickname
+ }
+ if req.Logo != nil {
+ org.Logo = *req.Logo
+ }
+ if req.Homepage != nil {
+ org.Homepage = *req.Homepage
+ }
+ if req.Verified != nil {
+ org.Verified = *req.Verified
+ }
+ if req.OrgType != nil {
+ org.OrgType = *req.OrgType
+ }
+ err = c.orgStore.Update(ctx, &org)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update database organization, error: %w", err)
+ }
+
+ //skip update git server
+ if req.Nickname == nil && req.Description == nil {
+ return &org, nil
+ }
+ var gitEditReq types.EditOrgReq
+ gitEditReq.Name = org.Name
+ gitEditReq.Nickname = &org.Nickname
+ gitEditReq.Description = &org.Description
+ _, err = c.gs.UpdateOrganization(&gitEditReq, nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to update git organization, error: %w", err)
+ }
+ return &org, err
+}
+
+
+
package component
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/bwmarrin/snowflake"
+ "github.com/casdoor/casdoor-go-sdk/casdoorsdk"
+ "github.com/google/uuid"
+ "opencsg.com/csghub-server/builder/git"
+ "opencsg.com/csghub-server/builder/git/gitserver"
+ "opencsg.com/csghub-server/builder/store/database"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+)
+
+const GitalyRepoNotFoundErr = "rpc error: code = NotFound desc = repository does not exist"
+
+type userComponentImpl struct {
+ userStore database.UserStore
+ orgStore database.OrgStore
+ nsStore database.NamespaceStore
+ repo database.RepoStore
+ ds database.DeployTaskStore
+ ams database.AccountMeteringStore
+ asqs database.AccountSyncQuotaStore
+ aus database.AccountUserStore
+
+ gs gitserver.GitServer
+ jwtc JwtComponent
+ tokenc AccessTokenComponent
+
+ casc *casdoorsdk.Client
+ casConfig *casdoorsdk.AuthConfig
+ once *sync.Once
+ sfnode *snowflake.Node
+ config *config.Config
+}
+
+type UserComponent interface {
+ ChangeUserName(ctx context.Context, oldUserName, newUserName, opUser string) error
+ Update(ctx context.Context, req *types.UpdateUserRequest, opUser string) error
+ Delete(ctx context.Context, operator, username string) error
+ // CanAdmin checks if a user has admin privileges.
+ //
+ // Parameters:
+ // - ctx: The context.Context object for the function.
+ // - username: The username of the user to check.
+ //
+ // Returns:
+ // - bool: True if the user has admin privileges, false otherwise.
+ // - error: An error if the user cannot be found in the database.
+ CanAdmin(ctx context.Context, username string) (bool, error)
+ // GetInternal get *full* user info by username or uuid
+ //
+ // should only be called by other *internal* services
+ GetInternal(ctx context.Context, userNameOrUUID string, useUUID bool) (*types.User, error)
+ Get(ctx context.Context, userNameOrUUID, visitorName string, useUUID bool) (*types.User, error)
+ CheckOperatorAndUser(ctx context.Context, operator, username string) (bool, error)
+ CheckIfUserHasOrgs(ctx context.Context, userName string) (bool, error)
+ CheckIffUserHasRunningOrBuildingDeployments(ctx context.Context, userName string) (bool, error)
+ CheckIfUserHasBills(ctx context.Context, userName string) (bool, error)
+ Index(ctx context.Context, visitorName, search string, per, page int) ([]*types.User, int, error)
+ Signin(ctx context.Context, code, state string) (*types.JWTClaims, string, error)
+ FixUserData(ctx context.Context, userName string) error
+}
+
+func NewUserComponent(config *config.Config) (UserComponent, error) {
+ var err error
+ c := &userComponentImpl{}
+ c.userStore = database.NewUserStore()
+ c.orgStore = database.NewOrgStore()
+ c.nsStore = database.NewNamespaceStore()
+ c.repo = database.NewRepoStore()
+ c.ds = database.NewDeployTaskStore()
+ c.ams = database.NewAccountMeteringStore()
+ c.asqs = database.NewAccountSyncQuotaStore()
+ c.aus = database.NewAccountUserStore()
+ c.jwtc = NewJwtComponent(config.JWT.SigningKey, config.JWT.ValidHour)
+ c.tokenc, err = NewAccessTokenComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("fail to create access token component, error: %w", err)
+ }
+ c.gs, err = git.NewGitServer(config)
+ if err != nil {
+ newError := fmt.Errorf("failed to create git server,error:%w", err)
+ return nil, newError
+ }
+ c.once = new(sync.Once)
+
+ certData, err := os.ReadFile(config.Casdoor.Certificate)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read casdoor certificate file,error:%w", err)
+ }
+ c.casConfig = &casdoorsdk.AuthConfig{
+ Endpoint: config.Casdoor.Endpoint,
+ ClientId: config.Casdoor.ClientID,
+ ClientSecret: config.Casdoor.ClientSecret,
+ Certificate: string(certData),
+ OrganizationName: config.Casdoor.OrganizationName,
+ ApplicationName: config.Casdoor.ApplicationName,
+ }
+ c.config = config
+
+ return c, nil
+}
+
+// // This function creates a user when user register from portal, without casdoor
+// func (c *userComponentImpl) createFromPortalRegistry(ctx context.Context, req types.CreateUserRequest) (*database.User, error) {
+// // Panic if the function has not been implemented
+// panic("implement me later")
+// }
+
+func (c *userComponentImpl) createFromCasdoorUser(ctx context.Context, cu casdoorsdk.User) (*database.User, error) {
+ var (
+ gsUserResp *gitserver.CreateUserResponse
+ err error
+ userName string
+ email string
+ canChangeUserName bool
+ )
+ //wechat user need to change username later
+ if cu.WeChat != "" {
+ userName, err = c.genUniqueName()
+ if err != nil {
+ return nil, fmt.Errorf("failed to generate unique user name,error:%w", err)
+ }
+ canChangeUserName = true
+ //set email to "", make sure not to create git user
+ email = ""
+ } else {
+ userName = cu.Name
+ canChangeUserName = false
+ email = cu.Email
+ }
+ //skip creating git user if email is empty, it will be created later when user set email
+ if email != "" {
+ gsUserReq := gitserver.CreateUserRequest{
+ Nickname: userName,
+ Username: userName,
+ Email: email,
+ }
+ gsUserResp, err = c.gs.CreateUser(gsUserReq)
+ if err != nil {
+ newError := fmt.Errorf("failed to create gitserver user '%s',error:%w", cu.Name, err)
+ return nil, newError
+ }
+ }
+
+ namespace := &database.Namespace{
+ Path: userName,
+ }
+ user := &database.User{
+ Username: userName,
+ NickName: userName,
+ Email: email,
+ UUID: cu.Id,
+ RegProvider: "casdoor",
+ Gender: cu.Gender,
+ // RoleMask: "", //will be updated when admin set user role
+ Phone: cu.Phone,
+ PhoneVerified: false,
+ EmailVerified: false,
+ LastLoginAt: cu.LastSigninTime,
+ Avatar: cu.Avatar,
+ CompanyVerified: false,
+ // PasswordHash: cu.Password,
+ Homepage: cu.Homepage,
+ Bio: cu.Bio,
+ CanChangeUserName: canChangeUserName,
+ }
+ if gsUserResp != nil {
+ user.GitID = gsUserResp.GitID
+ user.Password = gsUserResp.Password
+ }
+ err = c.userStore.Create(ctx, user, namespace)
+ if err != nil {
+ newError := fmt.Errorf("failed to create user in db,error:%w", err)
+ return nil, newError
+ }
+
+ return user, nil
+}
+
+func (c *userComponentImpl) ChangeUserName(ctx context.Context, oldUserName, newUserName, opUser string) error {
+ if oldUserName != opUser {
+ return fmt.Errorf("user name can only be changed by user self, user: '%s', op user: '%s'", oldUserName, opUser)
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, oldUserName)
+ if err != nil {
+ return fmt.Errorf("failed to find user by old name in db,error:%w", err)
+ }
+
+ if !user.CanChangeUserName {
+ return fmt.Errorf("user name can not be changed")
+ }
+
+ newUser, err := c.userStore.FindByUsername(ctx, newUserName)
+ if err != nil && !errors.Is(err, sql.ErrNoRows) {
+ return fmt.Errorf("failed to find user by new name in db,error:%w", err)
+ }
+ if newUser.ID > 0 {
+ return fmt.Errorf("user name '%s' already exists", newUserName)
+ }
+
+ err = c.userStore.ChangeUserName(ctx, oldUserName, newUserName)
+ if err != nil {
+ return fmt.Errorf("failed to change user name in db,error:%w", err)
+ }
+
+ //skip casdoor update if it's not a casdoor user
+ if user.UUID == "" || user.RegProvider != "casdoor" {
+ return nil
+ }
+
+ c.lazyInit()
+
+ err = c.updateCasdoorUser(&types.UpdateUserRequest{
+ UUID: &user.UUID,
+ NewUserName: &newUserName,
+ })
+ if err != nil {
+ newError := fmt.Errorf("failed to update casdoor user, uuid:'%s',error:%w", user.UUID, err)
+ return newError
+ }
+ return nil
+}
+
+func (c *userComponentImpl) Update(ctx context.Context, req *types.UpdateUserRequest, opUser string) error {
+ c.lazyInit()
+
+ user, err := c.userStore.FindByUsername(ctx, req.Username)
+ if err != nil {
+ newError := fmt.Errorf("failed to find user by name in db,error:%w", err)
+ return newError
+ }
+ if req.Roles != nil && (opUser == "" || opUser == req.Username) {
+ return fmt.Errorf("need another user to change roles of user '%s'", req.Username)
+ }
+ // need at least admin permission to update other user's info
+ if req.Username != opUser {
+ opuser, err := c.userStore.FindByUsername(ctx, opUser)
+ if err != nil {
+ return fmt.Errorf("failed to find op user by name in db,user: '%s', error:%w", opUser, err)
+ }
+ //check whether user has admin permission
+ canAdmin := opuser.CanAdmin()
+ if !canAdmin {
+ return fmt.Errorf("failed to update user '%s', op user '%s' is not admin", req.Username, opUser)
+ }
+ }
+
+ if req.Email != nil {
+ err = c.upsertGitUser(user.Username, req.Nickname, user.Email, *req.Email)
+ if err != nil {
+ return err
+ }
+ }
+
+ c.setChangedProps(&user, req)
+ err = c.userStore.Update(ctx, &user)
+ if err != nil {
+ newError := fmt.Errorf("failed to update database user '%s',error:%w", req.Username, err)
+ return newError
+ }
+
+ //skip casdoor update if it's not a casdoor user
+ if req.UUID == nil || user.RegProvider != "casdoor" {
+ return nil
+ }
+ err = c.updateCasdoorUser(req)
+ if err != nil {
+ newError := fmt.Errorf("failed to update casdoor user '%s',error:%w", req.Username, err)
+ return newError
+ }
+
+ return nil
+}
+
+// user registry with wechat does not have email, so git user is not created after signin
+// when user set email, a git user needs to be created
+func (c *userComponentImpl) upsertGitUser(username string, nickname *string, oldEmail, newEmail string) error {
+ var err error
+ if nickname == nil {
+ nickname = &username
+ }
+ if oldEmail == "" {
+ // create git user
+ gsUserReq := gitserver.CreateUserRequest{
+ Nickname: *nickname,
+ Username: username,
+ Email: newEmail,
+ }
+ _, err = c.gs.CreateUser(gsUserReq)
+ if err != nil {
+ newError := fmt.Errorf("failed to create git user '%s',error:%w", username, err)
+ return newError
+ }
+ } else {
+ // update git user
+ err = c.gs.UpdateUserV2(gitserver.UpdateUserRequest{
+ Nickname: nickname,
+ Username: username,
+ Email: &newEmail,
+ })
+ if err != nil {
+ newError := fmt.Errorf("failed to update git user '%s',error:%w", username, err)
+ return newError
+ }
+ }
+
+ return nil
+}
+
+func (c *userComponentImpl) setChangedProps(user *database.User, req *types.UpdateUserRequest) {
+ if req.Email != nil {
+ user.Email = *req.Email
+ if user.CanChangeUserName {
+ user.CanChangeUserName = false
+ slog.Info("use set email, disallow to change user name later (can_change_user_name=false)",
+ slog.String("username", user.Username), slog.String("email", user.Email))
+ }
+ }
+ if req.UUID != nil {
+ user.UUID = *req.UUID
+ }
+ if req.Avatar != nil {
+ user.Avatar = *req.Avatar
+ }
+ if req.Bio != nil {
+ user.Bio = *req.Bio
+ }
+ if req.Homepage != nil {
+ user.Homepage = *req.Homepage
+ }
+ if req.Phone != nil {
+ user.Phone = *req.Phone
+ }
+ if req.Nickname != nil {
+ user.NickName = *req.Nickname
+ }
+ if req.Roles != nil {
+ user.SetRoles(*req.Roles)
+ }
+}
+
+func (c *userComponentImpl) Delete(ctx context.Context, operator, username string) error {
+ user, err := c.userStore.FindByUsername(ctx, username)
+ if err != nil {
+ newError := fmt.Errorf("failed to find user by name in db,error:%w", err)
+ return newError
+ }
+
+ // TODO:delete user from git server
+ slog.Debug("delete user from git server", slog.String("operator", operator), slog.String("username", user.Username))
+
+ // if c.config.GitServer.Type == types.GitServerTypeGitea {
+ // // gitea gitserver does not support delete user, you could create a pr to our repo to fix it
+ // }
+
+ if c.config.GitServer.Type == types.GitServerTypeGitaly {
+ repos, err := c.repo.ByUser(ctx, user.ID)
+ if err != nil {
+ slog.Error("failed to find all repos for user", slog.String("username", user.Username), slog.Any("error", err))
+ return fmt.Errorf("failed to find all repos for user: %v", err)
+ }
+
+ for _, repo := range repos {
+ namespaceAndName := strings.Split(repo.Path, "/")
+ err := c.gs.DeleteRepo(ctx, gitserver.DeleteRepoReq{
+ Namespace: namespaceAndName[0],
+ Name: namespaceAndName[1],
+ RepoType: repo.RepositoryType,
+ })
+ if err != nil && err.Error() != GitalyRepoNotFoundErr {
+ slog.Error("failed to delete user repos in git server", slog.String("username", user.Username), slog.String("repo_path", repo.Path), slog.Any("error", err))
+ return fmt.Errorf("failed to delete user repos in git server: %v", err)
+ }
+ }
+ }
+ // delete user from db
+ err = c.userStore.DeleteUserAndRelations(ctx, user)
+ if err != nil {
+ return fmt.Errorf("failed to delete user and user relations: %v", err)
+ }
+
+ // delete user from casdoor
+ if user.UUID != "" {
+ casUser := &casdoorsdk.User{Id: user.UUID}
+ _, err = c.casc.DeleteUser(casUser)
+ return fmt.Errorf("failed to delete user in casdoor: %v", err)
+ }
+ return err
+}
+
+// CanAdmin checks if a user has admin privileges.
+//
+// Parameters:
+// - ctx: The context.Context object for the function.
+// - username: The username of the user to check.
+//
+// Returns:
+// - bool: True if the user has admin privileges, false otherwise.
+// - error: An error if the user cannot be found in the database.
+func (c *userComponentImpl) CanAdmin(ctx context.Context, username string) (bool, error) {
+ user, err := c.userStore.FindByUsername(ctx, username)
+ if err != nil {
+ newError := fmt.Errorf("failed to find user by name '%s' in db,error:%w", username, err)
+ return false, newError
+ }
+ return user.CanAdmin(), nil
+}
+
+// GetInternal get *full* user info by username or uuid
+//
+// should only be called by other *internal* services
+func (c *userComponentImpl) GetInternal(ctx context.Context, userNameOrUUID string, useUUID bool) (*types.User, error) {
+ var dbuser = new(database.User)
+ var err error
+ if useUUID {
+ dbuser, err = c.userStore.FindByUUID(ctx, userNameOrUUID)
+ } else {
+ *dbuser, err = c.userStore.FindByUsername(ctx, userNameOrUUID)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user by name or uuid '%s' in db,error:%w", userNameOrUUID, err)
+ }
+ return c.buildUserInfo(ctx, dbuser, false)
+}
+
+func (c *userComponentImpl) Get(ctx context.Context, userNameOrUUID, visitorName string, useUUID bool) (*types.User, error) {
+ var dbuser = new(database.User)
+ var err error
+ if useUUID {
+ dbuser, err = c.userStore.FindByUUID(ctx, userNameOrUUID)
+ } else {
+ *dbuser, err = c.userStore.FindByUsername(ctx, userNameOrUUID)
+ }
+ if err != nil {
+ return nil, fmt.Errorf("failed to find user by name or uuid '%s' in db,error:%w", userNameOrUUID, err)
+ }
+ userName := dbuser.Username
+ var onlyBasicInfo bool
+ //allow anonymous user to get basic info
+ if visitorName == "" {
+ onlyBasicInfo = true
+ } else if userName != visitorName {
+ canAdmin, err := c.CanAdmin(ctx, visitorName)
+ if err != nil {
+ return nil, fmt.Errorf("failed to check visitor user permission, visitor: %s, error: %w", visitorName, err)
+ }
+
+ if !canAdmin {
+ onlyBasicInfo = true
+ }
+ }
+
+ return c.buildUserInfo(ctx, dbuser, onlyBasicInfo)
+}
+
+func (c *userComponentImpl) CheckOperatorAndUser(ctx context.Context, operator, username string) (bool, error) {
+ opUser, err := c.userStore.FindByUsername(ctx, operator)
+ if err != nil {
+ newError := fmt.Errorf("failed to find operator by name in db,error:%w", err)
+ return true, newError
+ }
+
+ user, err := c.userStore.FindByUsername(ctx, username)
+ if err != nil {
+ newError := fmt.Errorf("failed to find user by name in db,error:%w", err)
+ return true, newError
+ }
+ if !opUser.CanAdmin() {
+ return false, errors.New("only admin user or the user can delete the user")
+ }
+
+ if user.CanAdmin() {
+ return false, errors.New("admin user can not be deleted")
+ }
+ return false, nil
+}
+
+func (c *userComponentImpl) CheckIfUserHasOrgs(ctx context.Context, userName string) (bool, error) {
+ var (
+ err error
+ total int
+ )
+ if _, total, err = c.orgStore.GetUserOwnOrgs(ctx, userName); err != nil {
+ return false, fmt.Errorf("failed to find orgs by username in db,error:%w", err)
+ }
+ return total > 0, nil
+}
+
+func (c *userComponentImpl) CheckIffUserHasRunningOrBuildingDeployments(ctx context.Context, userName string) (bool, error) {
+ user, err := c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return false, fmt.Errorf("failed to find user by username in db, error: %v", err)
+ }
+ deploys, err := c.ds.ListAllDeployments(ctx, user.ID)
+ if err != nil {
+ return false, fmt.Errorf("failed to list all deployments for user %s in db, error: %v", userName, err)
+ }
+ if len(deploys) > 0 {
+ return true, nil
+ }
+ return false, nil
+}
+
+func (c *userComponentImpl) CheckIfUserHasBills(ctx context.Context, userName string) (bool, error) {
+ user, err := c.userStore.FindByUsername(ctx, userName)
+ if err != nil {
+ return false, fmt.Errorf("failed to find user by username in db, error: %v", err)
+ }
+ ams, err := c.ams.ListAllByUserUUID(ctx, user.UUID)
+ if err != nil {
+ return false, fmt.Errorf("failed to list all account meterings for user %s in db, error: %w", userName, err)
+ }
+ if len(ams) > 0 {
+ return true, nil
+ }
+
+ asqs, err := c.asqs.ListAllByUserID(ctx, user.ID)
+ if err != nil {
+ return false, fmt.Errorf("failed to list all account sync quotas for user %s in db, error: %w", userName, err)
+ }
+ if len(asqs) > 0 {
+ return true, nil
+ }
+
+ aus, err := c.aus.ListAllByUserUUID(ctx, user.UUID)
+ if err != nil {
+ return false, fmt.Errorf("failed to list all account users for user %s in db, error: %w", userName, err)
+ }
+ if len(aus) > 0 {
+ return true, nil
+ }
+
+ return false, nil
+}
+
+func (c *userComponentImpl) buildUserInfo(ctx context.Context, dbuser *database.User, onlyBasicInfo bool) (*types.User, error) {
+ u := types.User{
+ Username: dbuser.Username,
+ Nickname: dbuser.NickName,
+ Avatar: dbuser.Avatar,
+ }
+
+ if !onlyBasicInfo {
+ u.ID = dbuser.ID
+ u.Email = dbuser.Email
+ u.UUID = dbuser.UUID
+ u.Bio = dbuser.Bio
+ u.Homepage = dbuser.Homepage
+ u.Phone = dbuser.Phone
+ u.Roles = dbuser.Roles()
+ }
+
+ dborgs, err := c.orgStore.GetUserBelongOrgs(ctx, dbuser.ID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get orgs for user %s,error:%w", dbuser.Username, err)
+ }
+
+ if len(dborgs) > 0 {
+ for _, org := range dborgs {
+ u.Orgs = append(u.Orgs, types.Organization{
+ Name: org.Name,
+ Nickname: org.Nickname,
+ Homepage: org.Homepage,
+ Logo: org.Logo,
+ OrgType: org.OrgType,
+ Verified: org.Verified,
+ UserID: org.UserID,
+ })
+ }
+ }
+
+ return &u, nil
+}
+
+func (c *userComponentImpl) Index(ctx context.Context, visitorName, search string, per, page int) ([]*types.User, int, error) {
+ var (
+ respUsers []*types.User
+ onlyBasicInfo bool
+ )
+ canAdmin, err := c.CanAdmin(ctx, visitorName)
+ if err != nil {
+ return nil, 0, fmt.Errorf("failed to check visitor user permission, visitor: %s, error: %w", visitorName, err)
+ }
+ if !canAdmin {
+ onlyBasicInfo = true
+ }
+
+ dbusers, count, err := c.userStore.IndexWithSearch(ctx, search, per, page)
+ if err != nil {
+ newError := fmt.Errorf("failed to find user by name in db,error:%w", err)
+ return nil, count, newError
+ }
+
+ for _, dbuser := range dbusers {
+ user := &types.User{
+ Username: dbuser.Username,
+ Nickname: dbuser.NickName,
+ Avatar: dbuser.Avatar,
+ }
+
+ if !onlyBasicInfo {
+ user.Email = dbuser.Email
+ user.UUID = dbuser.UUID
+ user.Bio = dbuser.Bio
+ user.Homepage = dbuser.Homepage
+ user.Phone = dbuser.Phone
+ user.Roles = dbuser.Roles()
+ }
+
+ respUsers = append(respUsers, user)
+ }
+
+ return respUsers, count, nil
+}
+
+func (c *userComponentImpl) Signin(ctx context.Context, code, state string) (*types.JWTClaims, string, error) {
+ c.lazyInit()
+
+ casToken, err := c.casc.GetOAuthToken(code, state)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to get token from casdoor,error:%w", err)
+ }
+ claims, err := c.casc.ParseJwtToken(casToken.AccessToken)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to parse token from casdoor,error:%w", err)
+ }
+
+ cu := claims.User
+ exists, err := c.userStore.IsExistByUUID(ctx, cu.Id)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to check user existence by name in db,error:%w", err)
+ }
+
+ var dbu *database.User
+ if !exists {
+ dbu, err = c.createFromCasdoorUser(ctx, cu)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to create user,error:%w", err)
+ }
+ // auto create git access token for the new user
+ go func(username string) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ _, err := c.tokenc.Create(ctx, &types.CreateUserTokenRequest{
+ Username: username,
+ TokenName: uuid.NewString(),
+ Application: types.AccessTokenAppGit,
+ })
+ if err != nil {
+ slog.Error("failed to create git user access token", "error", err, "username", dbu.Username)
+ }
+ }(dbu.Username)
+ } else {
+ // get user from db for username, as casdoor may have different username
+ dbu, err = c.userStore.FindByUUID(ctx, cu.Id)
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to find user by uuid in db, uuid:%s, error:%w", cu.Id, err)
+ }
+ // update user login time asynchronously
+ go func() {
+ dbu.LastLoginAt = time.Now().Format("2006-01-02 15:04:05")
+ err := c.userStore.Update(ctx, dbu)
+ if err != nil {
+ slog.Error("failed to update user login time", "error", err, "username", dbu.Username)
+ }
+ }()
+ }
+ hubToken, signed, err := c.jwtc.GenerateToken(ctx, types.CreateJWTReq{
+ UUID: dbu.UUID,
+ })
+ if err != nil {
+ return nil, "", fmt.Errorf("failed to generate jwt token,error:%w", err)
+ }
+
+ return hubToken, signed, nil
+}
+
+func (c *userComponentImpl) genUniqueName() (string, error) {
+ c.lazyInit()
+
+ if c.sfnode == nil {
+ return "", fmt.Errorf("user component sfnode is nil")
+ }
+ id := c.sfnode.Generate().Base36()
+ return "user_" + id, nil
+}
+
+func (c *userComponentImpl) updateCasdoorUser(req *types.UpdateUserRequest) error {
+ c.lazyInit()
+
+ casu, err := c.casc.GetUserByUserId(*req.UUID)
+ if err != nil {
+ return fmt.Errorf("failed to get user from casdoor,error:%w", err)
+ }
+ if casu == nil {
+ return fmt.Errorf("user not exists in casdoor")
+ }
+ var cols []string
+ if req.Email != nil {
+ casu.Email = *req.Email
+ cols = append(cols, "email")
+ }
+ if req.Phone != nil {
+ casu.Phone = *req.Phone
+ cols = append(cols, "phone")
+ }
+
+ if len(cols) == 0 {
+ return nil
+ }
+
+ // casdoor update user api don't allow empty display name, so we set it but not update it
+ if casu.DisplayName == "" {
+ casu.DisplayName = casu.Name
+ }
+
+ _, err = c.casc.UpdateUserForColumns(casu, cols)
+ return err
+}
+
+func (c *userComponentImpl) lazyInit() {
+ c.once.Do(func() {
+ var err error
+ c.casc = casdoorsdk.NewClientWithConf(c.casConfig)
+ c.sfnode, err = snowflake.NewNode(1)
+ if err != nil {
+ slog.Error("failed to create snowflake node", "error", err)
+ }
+ })
+}
+
+func (c *userComponentImpl) FixUserData(ctx context.Context, userName string) error {
+ err := c.gs.FixUserData(ctx, userName)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ apicomponent "opencsg.com/csghub-server/component"
+ "opencsg.com/csghub-server/user/component"
+)
+
+func NewAccessTokenHandler(config *config.Config) (*AccessTokenHandler, error) {
+ ac, err := component.NewAccessTokenComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := apicomponent.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &AccessTokenHandler{
+ c: ac,
+ sc: sc,
+ }, nil
+}
+
+type AccessTokenHandler struct {
+ c component.AccessTokenComponent
+ sc apicomponent.SensitiveComponent
+}
+
+// CreateAccessToken godoc
+// @Security ApiKey
+// @Summary [Deprecated: use POST:/token/{app}/{username} instead]
+// @Description create access token for a user
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param current_user query string true "current user, the owner"
+// @Param body body types.CreateUserTokenRequest true "body"
+// @Success 200 {object} types.Response{data=database.AccessToken} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/tokens [post]
+func (h *AccessTokenHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.CreateUserTokenRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var err error
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ if req.Application == "" {
+ req.Application = types.AccessTokenAppGit
+ }
+
+ req.Username = ctx.Param("username")
+ if currentUser != req.Username {
+ slog.Error("user can only create its own access token", slog.String("current_user", currentUser), slog.String("username", req.Username))
+ httpbase.UnauthorizedError(ctx, errors.New("user can only create its own access token"))
+ return
+ }
+ token, err := h.c.Create(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create user access token", slog.String("user_name", req.Username), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, token)
+}
+
+// CreateAppToken godoc
+// @Security ApiKey
+// @Summary Create access token for an special application
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param token_name path string true "token name"
+// @Param app path string true "application" Enums(git,starship)
+// @Param current_user query string true "current user, the owner"
+// @Param body body types.CreateUserTokenRequest true "body"
+// @Success 200 {object} types.Response{data=database.AccessToken} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /token/{app}/{token_name} [post]
+func (h *AccessTokenHandler) CreateAppToken(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.CreateUserTokenRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var err error
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req.Application = types.AccessTokenApp(ctx.Param("app"))
+ req.Username = currentUser
+ req.TokenName = ctx.Param("token_name")
+ token, err := h.c.Create(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create user access token", slog.String("user_name", req.Username), slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, token)
+}
+
+// Deprecated: use DeleteAppToken instead
+// DeleteAccessToken godoc
+// @Security ApiKey
+// @Summary [Deprecated: use DELETE:/token/{app}/{token_name} instead]
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param token_name path string true "token_name"
+// @Param current_user query string true "current user, the owner"
+// @Param body body types.DeleteUserTokenRequest true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/tokens/{token_name} [delete]
+func (h *AccessTokenHandler) Delete(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ var req types.DeleteUserTokenRequest
+ req.Username = ctx.Param("username")
+ if currentUser != req.Username {
+ slog.Error("user can only delete its own access token", slog.String("current_user", currentUser), slog.String("username", req.Username))
+ httpbase.UnauthorizedError(ctx, errors.New("user can only delete its own access token"))
+ return
+ }
+ req.TokenName = ctx.Param("token_name")
+ err := h.c.Delete(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to delete user access token", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Delete user access token succeed")
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteAppToken godoc
+// @Security ApiKey
+// @Summary Delete access token of a app
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param app path string true "application" Enums(git,starship)
+// @Param token_name path string true "token_name"
+// @Param current_user query string true "current user, the owner"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /token/{app}/{token_name} [delete]
+func (h *AccessTokenHandler) DeleteAppToken(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ app := ctx.Param("app")
+ tokenName := ctx.Param("token_name")
+ req := types.DeleteUserTokenRequest{
+ Username: currentUser,
+ TokenName: tokenName,
+ Application: types.AccessTokenApp(app),
+ }
+ err := h.c.Delete(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to delete user access token", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Delete user access token succeed")
+ httpbase.OK(ctx, nil)
+}
+
+// RefreshToken godoc
+// @Security ApiKey
+// @Summary Refresh a access token for a user
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param app path string true "application" Enums(git,starship)
+// @Param token_name path string true "token_name"
+// @Param current_user query string true "current user, the owner"
+// @Param expired_at query string false "new expire time, in format RFC3339, like 2006-01-02T15:04:05Z07:00"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /token/{app}/{token_name} [put]
+func (h *AccessTokenHandler) Refresh(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ app := ctx.Param("app")
+ tokenName := ctx.Param("token_name")
+ var expiredAt time.Time
+ var err error
+ paramExpiredAt := ctx.Param("expired_at")
+ if len(paramExpiredAt) > 0 {
+ expiredAt, err = time.Parse(time.RFC3339, paramExpiredAt)
+ if err != nil {
+ slog.Error("Failed to parse expired_at", slog.String("expired_at", paramExpiredAt), slog.Any("error", err))
+ httpbase.BadRequest(ctx, "cannot parse expired_at, please use format RFC3339, like 2006-01-02T15:04:05Z07:00")
+ return
+ }
+ }
+ resp, err := h.c.RefreshToken(ctx, currentUser, tokenName, app, expiredAt)
+ if err != nil {
+ slog.Error("Failed to refresh user access token", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("refresh user access token succeed", slog.String("current_user", currentUser),
+ slog.String("app", app), slog.String("token_name", tokenName))
+ httpbase.OK(ctx, resp)
+}
+
+// GetAccessToken godoc
+// @Security ApiKey
+// @Summary Get token and owner's detail by the token value
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param token_value path string true "token_value"
+// @Param app query string false "application" Enums(git,starship)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /token/{token_value} [get]
+func (h *AccessTokenHandler) Get(ctx *gin.Context) {
+ //!can not check current user here, because dont know user name when validate an access token
+ /*
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ */
+ var req types.CheckAccessTokenReq
+ req.Token = ctx.Param("token_value")
+ req.Application = ctx.Query("app")
+ resp, err := h.c.Check(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to check user access token", slog.Any("error", err), slog.Any("req", req))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, resp)
+}
+
+// GetUserTokens godoc
+// @Security ApiKey
+// @Summary Get all access tokens for a user
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param current_user query string false "current user name"
+// @Param app query string false "application" Enums(git,starship)
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/tokens [get]
+func (h *AccessTokenHandler) GetUserTokens(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ app := ctx.Query("app")
+ resp, err := h.c.GetTokens(ctx, currentUser, app)
+ if err != nil {
+ slog.Error("Failed to get user access tokens", slog.Any("error", err), slog.Any("application", app), slog.String("current_user", currentUser))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, resp)
+}
+
+// GetUserFirstToken godoc
+// @Security ApiKey
+// @Summary Get or create first available access token for a user
+// @Tags Access token
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param current_user query string false "current user name"
+// @Param app query string false "application" Enums(git,starship)
+// @Param token_name query string false "token name"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username}/tokens/first [get]
+func (h *AccessTokenHandler) GetOrCreateFirstAvaiTokens(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, component.ErrUserNotFound)
+ return
+ }
+ app := ctx.Query("app")
+ tokenName := ctx.Query("token_name")
+ if app == "" || tokenName == "" {
+ httpbase.BadRequest(ctx, "app and tokenName query parameters are required")
+ return
+ }
+ resp, err := h.c.GetOrCreateFirstAvaiToken(ctx, currentUser, app, tokenName)
+ if err != nil {
+ slog.Error("Failed to get user access tokens", slog.Any("error", err), slog.Any("application", app), slog.String("current_user", currentUser))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, resp)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/user/component"
+)
+
+func NewJWTHandler(config *config.Config) (*JWTHandler, error) {
+ return &JWTHandler{
+ c: component.NewJwtComponent(config.JWT.SigningKey, config.JWT.ValidHour),
+ }, nil
+}
+
+type JWTHandler struct {
+ c component.JwtComponent
+}
+
+// CreateJWTToken godoc
+// @Security ApiKey
+// @Summary generate jwt token for user
+// @Tags JWT
+// @Accept json
+// @Produce json
+// @Param body body types.CreateJWTReq true "body"
+// @Success 200 {object} types.CreateJWTResp "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /jwt/token [post]
+func (h *JWTHandler) Create(ctx *gin.Context) {
+ var req types.CreateJWTReq
+ if err := ctx.ShouldBind(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ slog.Info("Create JWT token", "req", req)
+ claims, signed, err := h.c.GenerateToken(ctx.Request.Context(), req)
+ if err != nil {
+ slog.Error("failed to generate JWT token", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ expireTime := claims.ExpiresAt
+ resp := &types.CreateJWTResp{
+ Token: signed,
+ ExpireAt: expireTime.Time,
+ }
+
+ httpbase.OK(ctx, resp)
+}
+
+// VerifyJWTToken godoc
+// @Security ApiKey
+// @Summary verify jwt token and return user info
+// @Tags JWT
+// @Accept json
+// @Produce json
+// @Param token path string true "token"
+// @Success 200 {object} types.User "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /jwt/{token} [get]
+func (h *JWTHandler) Verify(ctx *gin.Context) {
+ token := ctx.Param("token")
+ user, err := h.c.ParseToken(ctx.Request.Context(), token)
+ if err != nil {
+ slog.Error("failed to verify JWT token", slog.Any("error", err), slog.String("token", token))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to verify JWT token '%s': %w", token, err))
+ return
+ }
+
+ httpbase.OK(ctx, user)
+}
+
+
+
package handler
+
+import (
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/utils/common"
+ "opencsg.com/csghub-server/user/component"
+)
+
+type MemberHandler struct {
+ c component.MemberComponent
+}
+
+func NewMemberHandler(config *config.Config) (*MemberHandler, error) {
+ mc, err := component.NewMemberComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &MemberHandler{
+ c: mc,
+ }, nil
+}
+
+// GetOrganizationMembers godoc
+// @Security ApiKey
+// @Summary Get organization members. Org member can get more details.
+// @Tags Member
+// @Accept json
+// @Produce json
+// @Param current_user query string false "the op user"
+// @param namespace path string true "namespace"
+// @Param per query int false "per" default(50)
+// @Param page query int false "per page" default(1)
+// @Success 200 {object} types.ResponseWithTotal{data=types.Member, total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/members [get]
+func (h *MemberHandler) OrgMembers(ctx *gin.Context) {
+ orgName := ctx.Param("namespace")
+ if orgName == "" {
+ httpbase.BadRequest(ctx, fmt.Errorf("org name is empty").Error())
+ return
+ }
+ pageSize, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ members, total, err := h.c.OrgMembers(ctx.Request.Context(), orgName, currentUser, pageSize, page)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": members,
+ "total": total,
+ }
+ httpbase.OK(ctx, respData)
+}
+
+// UpdateMember godoc
+// @Security ApiKey
+// @Summary update user membership
+// @Tags Member
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param username path string true "user name"
+// @Param current_user query string false "the op user"
+// @Param body body handler.Update.updateMemberRequest true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/members/{username} [put]
+func (h *MemberHandler) Update(ctx *gin.Context) {
+ type updateMemberRequest struct {
+ OldRole string `json:"old_role" binding:"required"`
+ NewRole string `json:"new_role" binding:"required"`
+ }
+ req := new(updateMemberRequest)
+ if err := ctx.ShouldBindJSON(req); err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("failed to unmarshal request body,caused by:%w", err).Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ org := ctx.Param("namespace")
+ userName := ctx.Param("username")
+ err := h.c.ChangeMemberRole(ctx, org, userName, currentUser, req.OldRole, req.NewRole)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// Create godoc
+// @Security ApiKey
+// @Summary Create new membership between org and user
+// @Description user will be added to org with a role
+// @Tags Member
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param current_user query string false "the op user"
+// @Param body body handler.Create.addMemberRequest true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/members [post]
+func (h *MemberHandler) Create(ctx *gin.Context) {
+ type addMemberRequest struct {
+ // name of user will be added to the org as a member
+ Users string `json:"users" binding:"required" example:"user1,user2"`
+ Role string `json:"role" binding:"required"`
+ }
+ req := new(addMemberRequest)
+ if err := ctx.ShouldBindJSON(req); err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("failed to unmarshal request body,caused by:%w", err).Error())
+ return
+ }
+ users := strings.Split(req.Users, ",")
+ if len(users) == 0 {
+ httpbase.BadRequest(ctx, fmt.Errorf("user name is empty").Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ org := ctx.Param("namespace")
+ err := h.c.AddMembers(ctx, org, users, currentUser, req.Role)
+ if err != nil {
+ slog.ErrorContext(ctx, "create member fail", slog.Any("error", err),
+ slog.Group("request",
+ slog.String("org", org), slog.String("user", req.Users), slog.String("role", req.Role), slog.String("op_user", currentUser),
+ ),
+ )
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// Delete godoc
+// @Security ApiKey
+// @Summary Remove membership between org and user
+// @Description user's role will be remove from org
+// @Tags Member
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param username path string true "user name"
+// @Param current_user query string false "the op user"
+// @Param body body handler.Delete.removeMemberRequest true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/members/{username} [delete]
+func (h *MemberHandler) Delete(ctx *gin.Context) {
+ type removeMemberRequest struct {
+ Role string `json:"role" binding:"required"`
+ }
+ req := new(removeMemberRequest)
+ if err := ctx.ShouldBindJSON(req); err != nil {
+ httpbase.BadRequest(ctx, fmt.Errorf("failed to unmarshal request body,caused by:%w", err).Error())
+ return
+ }
+ currentUser := httpbase.GetCurrentUser(ctx)
+ org := ctx.Param("namespace")
+ userName := ctx.Param("username")
+ err := h.c.Delete(ctx, org, userName, currentUser, req.Role)
+ if err != nil {
+ slog.ErrorContext(ctx, "delete member fail", slog.Any("error", err),
+ slog.Group("request",
+ slog.String("org", org), slog.String("username", userName), slog.String("role", req.Role), slog.String("op_user", currentUser),
+ ),
+ )
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, nil)
+}
+
+// GetMemberRole godoc
+// @Security ApiKey
+// @Summary Get user's role in an org
+// @Tags Member
+// @Accept json
+// @Produce json
+// @Param namespace path string true "org name"
+// @Param username path string true "user name"
+// @Param current_user query string false "the op user"
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace}/members/{username} [get]
+func (h *MemberHandler) GetMemberRole(ctx *gin.Context) {
+ org := ctx.Param("namespace")
+ userName := ctx.Param("username")
+ // Assuming GetMemberRole returns a role (or similar) and an error
+ role, err := h.c.GetMemberRole(ctx.Request.Context(), org, userName)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ httpbase.OK(ctx, role)
+}
+
+
+
package handler
+
+import (
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/component"
+)
+
+type NamespaceHandler struct {
+ c component.NamespaceComponent
+}
+
+func NewNamespaceHandler(config *config.Config) (*NamespaceHandler, error) {
+ nc, err := component.NewNamespaceComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ return &NamespaceHandler{
+ c: nc,
+ }, nil
+}
+
+// GetInfo godoc
+// @Security ApiKey
+// @Summary Get namespace info [Internal Only].
+// @Description get namespace info
+// @Tags Namespace, InternalOnly
+// @Accept json
+// @Produce json
+// @Param path path string true "namespace"
+// @Success 200 {object} types.Response{data=types.Namespace} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /namespace/{path} [get]
+func (h *NamespaceHandler) GetInfo(ctx *gin.Context) {
+ path := ctx.Param("path")
+ ns, err := h.c.GetInfo(ctx, path)
+ if err != nil {
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ httpbase.OK(ctx, ns)
+}
+
+
+
package handler
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ apicomponent "opencsg.com/csghub-server/component"
+ "opencsg.com/csghub-server/user/component"
+)
+
+func NewOrganizationHandler(config *config.Config) (*OrganizationHandler, error) {
+ oc, err := component.NewOrganizationComponent(config)
+ if err != nil {
+ return nil, err
+ }
+ sc, err := apicomponent.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ return &OrganizationHandler{
+ c: oc,
+ sc: sc,
+ }, nil
+}
+
+type OrganizationHandler struct {
+ c component.OrganizationComponent
+ sc apicomponent.SensitiveComponent
+}
+
+// CreateOrganization godoc
+// @Security ApiKey
+// @Summary Create a new organization
+// @Description create a new organization
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param current_user query string false "the op user"
+// @param body body types.CreateOrgReq true "body"
+// @Success 200 {object} types.Response{data=types.Organization} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organizations [post]
+func (h *OrganizationHandler) Create(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ var req types.CreateOrgReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var err error
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ req.Username = currentUser
+ org, err := h.c.Create(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to create organization", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Create organization succeed", slog.String("org_path", org.Name))
+ httpbase.OK(ctx, org)
+}
+
+// GetOrganization godoc
+// @Security ApiKey
+// @Summary Get organization info
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param current_user query string false "the op user"
+// @param namespace path string true "namespace"
+// @Success 200 {object} types.Response{data=types.Organization} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace} [get]
+func (h *OrganizationHandler) Get(ctx *gin.Context) {
+ orgName := ctx.Param("namespace")
+ if len(orgName) == 0 {
+ httpbase.BadRequest(ctx, "organization name is empty")
+ return
+ }
+ org, err := h.c.Get(ctx, orgName)
+ if err != nil {
+ slog.Error("Failed to get organization", slog.Any("error", err), slog.String("org_path", orgName))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get organization succeed", slog.String("org_path", org.Name))
+ httpbase.OK(ctx, org)
+}
+
+// GetOrganizations godoc
+// @Security ApiKey
+// @Summary Get organizations
+// @Description get organizations
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Success 200 {object} types.Response{data=[]types.Organization} "OK"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organizations [get]
+func (h *OrganizationHandler) Index(ctx *gin.Context) {
+ username := httpbase.GetCurrentUser(ctx)
+ search := ctx.Query("search")
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Failed to get per and page", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ orgs, total, err := h.c.Index(ctx, username, search, per, page)
+ if err != nil {
+ slog.Error("Failed to get organizations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ respData := gin.H{
+ "data": orgs,
+ "total": total,
+ }
+
+ slog.Info("Get organizations succeed", slog.String("username", username), slog.String("search", search), slog.Int("per", per), slog.Int("page", page))
+ httpbase.OK(ctx, respData)
+}
+
+// DeleteOrganization godoc
+// @Security ApiKey
+// @Summary Delete organization
+// @Description delete organization
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param current_user query string false "the op user"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace} [delete]
+func (h *OrganizationHandler) Delete(ctx *gin.Context) {
+ var req types.DeleteOrgReq
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ req.CurrentUser = currentUser
+ req.Name = ctx.Param("namespace")
+ err := h.c.Delete(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to delete organizations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Delete organizations succeed", slog.String("org_name", req.Name))
+ httpbase.OK(ctx, nil)
+}
+
+// UpdateOrganization godoc
+// @Security ApiKey
+// @Summary Update organization
+// @Description update organization
+// @Tags Organization
+// @Accept json
+// @Produce json
+// @Param namespace path string true "namespace"
+// @Param current_user query string false "the op user"
+// @Param body body types.EditOrgReq true "body"
+// @Success 200 {object} types.Response{data=database.Organization} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /organization/{namespace} [put]
+func (h *OrganizationHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("user not found, please login first"))
+ return
+ }
+
+ var req types.EditOrgReq
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ var err error
+ _, err = h.sc.CheckRequestV2(ctx, &req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+ req.CurrentUser = currentUser
+ req.Name = ctx.Param("namespace")
+ org, err := h.c.Update(ctx, &req)
+ if err != nil {
+ slog.Error("Failed to update organizations", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Update organizations succeed", slog.String("org_name", org.Nickname))
+ httpbase.OK(ctx, org)
+}
+
+
+
package handler
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "net/http"
+ "net/url"
+
+ "github.com/gin-gonic/gin"
+ "go.temporal.io/sdk/client"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/common/types"
+ "opencsg.com/csghub-server/common/utils/common"
+ apicomponent "opencsg.com/csghub-server/component"
+ "opencsg.com/csghub-server/user/component"
+ "opencsg.com/csghub-server/user/workflow"
+ workflowCommon "opencsg.com/csghub-server/user/workflow/common"
+)
+
+type UserHandler struct {
+ c component.UserComponent
+ sc apicomponent.SensitiveComponent
+ publicDomain string
+ EnableHTTPS bool
+ signinSuccessRedirectURL string
+ atc component.AccessTokenComponent
+ codeSoulerVScodeRedirectURL string
+ codeSoulerJetbrainsRedirectURL string
+ config *config.Config
+}
+
+const (
+ VSCODE = "vscode"
+ JETBRAINS = "jetbrains"
+)
+
+func NewUserHandler(config *config.Config) (*UserHandler, error) {
+ h := &UserHandler{}
+ var err error
+ h.c, err = component.NewUserComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create user component: %w", err)
+ }
+ sc, err := apicomponent.NewSensitiveComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating sensitive component:%w", err)
+ }
+ h.sc = sc
+ domainParsedUrl, err := url.Parse(config.APIServer.PublicDomain)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse public domain '%s': %w", config.APIServer.PublicDomain, err)
+ }
+ h.atc, err = component.NewAccessTokenComponent(config)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create access token component: %w", err)
+ }
+ h.publicDomain = domainParsedUrl.Hostname()
+ h.EnableHTTPS = config.EnableHTTPS
+ h.signinSuccessRedirectURL = config.User.SigninSuccessRedirectURL
+ h.codeSoulerVScodeRedirectURL = config.User.CodeSoulerVScodeRedirectURL
+ h.codeSoulerJetbrainsRedirectURL = config.User.CodeSoulerJetBrainsRedirectURL
+ h.config = config
+ return h, err
+}
+
+// CreateUser godoc
+// @Security ApiKey
+// @Summary Create a new user
+// @Description create a new user
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param body body types.CreateUserRequest true "body"
+// @Success 200 {object} types.Response{data=database.User} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /users [post]
+// func (h *UserHandler) Create(ctx *gin.Context) {
+// var req *types.CreateUserRequest
+// if err := ctx.ShouldBindJSON(&req); err != nil {
+// slog.Error("Bad request format", "error", err)
+// httpbase.BadRequest(ctx, err.Error())
+// return
+// }
+
+// slog.Debug("Creating user", slog.Any("req", req))
+// user, err := h.c.Create(ctx, req)
+// if err != nil {
+// slog.Error("Failed to create user", slog.Any("error", err))
+// httpbase.ServerError(ctx, err)
+// return
+// }
+
+// slog.Info("Create user succeed", slog.String("user", user.Username))
+// httpbase.OK(ctx, user)
+// }
+
+// UpdateUser godoc
+// @Security ApiKey
+// @Summary Update user. If change user name, should only send 'new_username' in the request body.
+// @Description update user
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param current_user query string true "current user"
+// @Param body body types.UpdateUserRequest true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username} [put]
+func (h *UserHandler) Update(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ var req *types.UpdateUserRequest
+ if err := ctx.ShouldBindJSON(&req); err != nil {
+ slog.Error("Bad request format", "error", err)
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ var err error
+ _, err = h.sc.CheckRequestV2(ctx, req)
+ if err != nil {
+ slog.Error("failed to check sensitive request", slog.Any("error", err))
+ httpbase.BadRequest(ctx, fmt.Errorf("sensitive check failed: %w", err).Error())
+ return
+ }
+
+ userName := ctx.Param("username")
+ req.Username = userName
+
+ if req.NewUserName != nil {
+ err = h.c.ChangeUserName(ctx, req.Username, *req.NewUserName, currentUser)
+ } else {
+ err = h.c.Update(ctx, req, currentUser)
+ }
+ if err != nil {
+ slog.Error("Failed to update user", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Update user succeed", slog.String("user", req.Username))
+ httpbase.OK(ctx, nil)
+}
+
+// DeleteUser godoc
+// @Security ApiKey
+// @Summary Delete user
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username"
+// @Param current_user query string true "current user"
+// @Param body body types.UpdateUserRequest true "body"
+// @Success 200 {object} types.Response{} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username} [delete]
+func (h *UserHandler) Delete(ctx *gin.Context) {
+ operator := httpbase.GetCurrentUser(ctx)
+ userName := ctx.Param("username")
+
+ // Check if operator can delete user
+ isServerErr, err := h.c.CheckOperatorAndUser(ctx, operator, userName)
+ if err != nil && isServerErr {
+ httpbase.ServerError(ctx, fmt.Errorf("user cannot be deleted: %w", err))
+ return
+ }
+ if err != nil && !isServerErr {
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+
+ // Check if user has organizations
+ hasOrgs, err := h.c.CheckIfUserHasOrgs(ctx, userName)
+ if err != nil {
+ httpbase.ServerError(ctx, fmt.Errorf("failed to check if user has organzitions, error: %w", err))
+ return
+ }
+ if hasOrgs {
+ httpbase.BadRequest(ctx, "users who own organizations cannot be deleted")
+ return
+ }
+ // Check if user has running or building deployments
+ hasDeployments, err := h.c.CheckIffUserHasRunningOrBuildingDeployments(ctx, userName)
+ if err != nil {
+ httpbase.ServerError(ctx, fmt.Errorf("failed to check if user has deployments, error: %w", err))
+ return
+ }
+ if hasDeployments {
+ httpbase.BadRequest(ctx, "users who own deployments cannot be deleted")
+ return
+ }
+
+ // Check if user has bills, Saas only
+ hasBills, err := h.c.CheckIfUserHasBills(ctx, userName)
+ if err != nil {
+ httpbase.ServerError(ctx, fmt.Errorf("failed to check if user has bills, error: %w", err))
+ return
+ }
+ if hasBills {
+ httpbase.BadRequest(ctx, "users who own bills cannot be deleted")
+ return
+ }
+
+ //start workflow to delete user
+ workflowClient := workflow.GetWorkflowClient()
+ workflowOptions := client.StartWorkflowOptions{
+ TaskQueue: workflow.WorkflowUserDeletionQueueName,
+ }
+
+ we, err := workflowClient.ExecuteWorkflow(context.Background(), workflowOptions, workflow.UserDeletionWorkflow,
+ workflowCommon.User{
+ Username: userName,
+ Operator: operator,
+ },
+ h.config,
+ )
+ if err != nil {
+ httpbase.ServerError(ctx, fmt.Errorf("failed to start user deletion workflow, error: %w", err))
+ return
+ }
+
+ slog.Info("start user deletion workflow", slog.String("workflow_id", we.GetID()), slog.String("userName", userName), slog.String("operator", operator))
+ httpbase.OK(ctx, nil)
+}
+
+// GetUser godoc
+// @Security ApiKey
+// @Summary Get user info. Admin and the user self can see full info, other users can only see basic info.
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param username path string true "username or uuid, defined by the query string 'type'"
+// @Param current_user query string false "current user"
+// @Param type query string false "path param is usernam or uuid, default to username" Enums(username, uuid)
+// @Success 200 {object} types.Response{data=types.User} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /user/{username} [get]
+func (h *UserHandler) Get(ctx *gin.Context) {
+ visitorName := httpbase.GetCurrentUser(ctx)
+ authType := httpbase.GetAuthType(ctx)
+ userNameOrUUID := ctx.Param("username")
+ useUUID := ctx.Query("type") == "uuid"
+ var user *types.User
+ var err error
+ if authType == httpbase.AuthTypeApiKey {
+ user, err = h.c.GetInternal(ctx, userNameOrUUID, useUUID)
+ } else {
+ user, err = h.c.Get(ctx, userNameOrUUID, visitorName, useUUID)
+ }
+ if err != nil {
+ slog.Error("Failed to get user", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+
+ slog.Info("Get user succeed", slog.String("userName", userNameOrUUID))
+ httpbase.OK(ctx, user)
+}
+
+// GetUsers godoc
+// @Security ApiKey
+// @Summary Get users info. Only Admin
+// @Tags User
+// @Accept json
+// @Produce json
+// @Param search query string true "search"
+// @Success 200 {object} types.Response{data=[]types.User,total=int} "OK"
+// @Failure 400 {object} types.APIBadRequest "Bad request"
+// @Failure 500 {object} types.APIInternalServerError "Internal server error"
+// @Router /users [get]
+func (h *UserHandler) Index(ctx *gin.Context) {
+ visitorName := httpbase.GetCurrentUser(ctx)
+ search := ctx.Query("search")
+ per, page, err := common.GetPerAndPageFromContext(ctx)
+ if err != nil {
+ slog.Error("Failed to get per and page", slog.Any("error", err))
+ httpbase.BadRequest(ctx, err.Error())
+ return
+ }
+ users, count, err := h.c.Index(ctx, visitorName, search, per, page)
+ if err != nil {
+ slog.Error("Failed to get user", slog.Any("error", err))
+ httpbase.ServerError(ctx, err)
+ return
+ }
+ respData := gin.H{
+ "data": users,
+ "total": count,
+ }
+
+ slog.Info("Get users succeed")
+ httpbase.OK(ctx, respData)
+}
+
+func (h *UserHandler) Casdoor(ctx *gin.Context) {
+ code := ctx.Query("code")
+ state := ctx.Query("state")
+ slog.Debug("get casdoor callback", slog.String("code", code), slog.String("state", state))
+
+ jwtToken, signed, err := h.c.Signin(ctx.Request.Context(), code, state)
+ if err != nil {
+ slog.Error("Failed to signin", slog.Any("error", err), slog.String("code", code), slog.String("state", state))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to signin: %w", err))
+ return
+ }
+
+ var (
+ targetUrl string
+ starshipApiKey string
+ )
+
+ if state == VSCODE || state == JETBRAINS {
+ starshipApiKey, err = h.getStarshipApiKey(ctx, jwtToken.CurrentUser, "codesouler-"+state)
+ if err != nil {
+ errMsg := fmt.Sprintf("failed to get user %s starship apikey for login from %s", jwtToken.CurrentUser, state)
+ slog.Error(errMsg, slog.String("code", code), slog.Any("error", err))
+ httpbase.ServerError(ctx, fmt.Errorf("failed to get apikey for %s", state))
+ return
+ }
+ codeSoulerEndpoint := h.codeSoulerVScodeRedirectURL
+ if state == JETBRAINS {
+ codeSoulerEndpoint = h.codeSoulerJetbrainsRedirectURL
+ }
+ targetUrl = fmt.Sprintf("%s?apikey=%s&portal_url=%s&jwt=%s", codeSoulerEndpoint, starshipApiKey, h.signinSuccessRedirectURL, signed)
+ } else {
+ targetUrl = fmt.Sprintf("%s?jwt=%s", h.signinSuccessRedirectURL, signed)
+ }
+
+ slog.Info("generate login redirect url", slog.Any("targetUrl", targetUrl))
+ ctx.Redirect(http.StatusMovedPermanently, targetUrl)
+}
+
+func (h *UserHandler) getStarshipApiKey(ctx *gin.Context, userName, tokenName string) (string, error) {
+ token, err := h.atc.GetOrCreateFirstAvaiToken(ctx, userName, string(types.AccessTokenAppStarship), tokenName)
+ if err != nil {
+ return "", fmt.Errorf("fail to get starship token for user %s, tokenName %s: %w", userName, tokenName, err)
+ }
+ return token, nil
+}
+
+
+
package router
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+
+ "github.com/gin-gonic/gin"
+ "opencsg.com/csghub-server/api/httpbase"
+ "opencsg.com/csghub-server/api/middleware"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/handler"
+)
+
+func NewRouter(config *config.Config) (*gin.Engine, error) {
+ r := gin.New()
+ r.Use(gin.Recovery())
+ r.Use(middleware.Log())
+ r.Use(middleware.Authenticator(config))
+
+ userHandler, err := handler.NewUserHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating user handler:%w", err)
+ }
+ acHandler, err := handler.NewAccessTokenHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating token handler:%w", err)
+ }
+ orgHandler, err := handler.NewOrganizationHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating user controller:%w", err)
+ }
+ // Member
+ memberCtrl, err := handler.NewMemberHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating user controller:%w", err)
+ }
+ //namespace
+ nsCtrl, err := handler.NewNamespaceHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating namespace controller:%w", err)
+ }
+
+ apiV1Group := r.Group("/api/v1")
+ jwtGroup := apiV1Group.Group("/jwt")
+ userGroup := apiV1Group.Group("/user")
+ tokenGroup := apiV1Group.Group("/token")
+
+ needAPIKey := middleware.OnlyAPIKeyAuthenticator(config)
+ jwtHandler, err := handler.NewJWTHandler(config)
+ if err != nil {
+ return nil, fmt.Errorf("error creating jwt handler:%w", err)
+ }
+
+ //don't need login
+ {
+ //casdoor
+ apiV1Group.GET("/callback/casdoor", userHandler.Casdoor)
+ //user
+ userGroup.GET("/:username", userHandler.Get)
+ userGroup.DELETE("/:username", userHandler.Delete)
+ // org and members
+ apiV1Group.GET("/organizations", orgHandler.Index)
+ apiV1Group.GET("/organization/:namespace", orgHandler.Get)
+ apiV1Group.GET("/organization/:namespace/members", memberCtrl.OrgMembers)
+ }
+
+ //internal only
+ {
+ //organization
+ //namespace
+ apiV1Group.GET("/namespace/:path", needAPIKey, nsCtrl.GetInfo)
+ //jwt
+ jwtGroup.POST("/token", needAPIKey, jwtHandler.Create)
+ jwtGroup.GET("/:token", needAPIKey, jwtHandler.Verify)
+ // check token info
+ tokenGroup.GET("/:token_value", needAPIKey, acHandler.Get)
+ }
+
+ apiV1Group.Use(mustLogin())
+ userMatch := userMatch()
+
+ // routers for users
+ {
+ // userGroup.POST("", userHandler.Create)
+ // user self or admin
+ userGroup.PUT("/:username", mustLogin(), userHandler.Update)
+ //TODO:
+ // userGroup.DELETE("/:username", userMatch, userHandler.Delete)
+ // get user's all tokens
+ userGroup.GET("/:username/tokens", userMatch, acHandler.GetUserTokens)
+ userGroup.GET("/:username/tokens/first", userMatch, acHandler.GetOrCreateFirstAvaiTokens)
+ // get user list
+ apiV1Group.GET("/users", mustLogin(), userHandler.Index)
+
+ }
+ // routers for organizations
+ {
+ apiV1Group.POST("/organizations", orgHandler.Create)
+ apiV1Group.PUT("/organization/:namespace", orgHandler.Update)
+ apiV1Group.DELETE("/organization/:namespace", orgHandler.Delete)
+ }
+ // routers for members
+ {
+ apiV1Group.GET("/organization/:namespace/members/:username", userMatch, memberCtrl.GetMemberRole)
+ apiV1Group.POST("/organization/:namespace/members", memberCtrl.Create)
+ apiV1Group.PUT("/organization/:namespace/members/:username", memberCtrl.Update)
+ apiV1Group.DELETE("/organization/:namespace/members/:username", memberCtrl.Delete)
+ }
+ // routers for access tokens
+ {
+ tokenGroup.POST("/:app/:token_name", acHandler.CreateAppToken)
+ tokenGroup.PUT("/:app/:token_name", acHandler.Refresh)
+ tokenGroup.DELETE("/:app/:token_name", acHandler.DeleteAppToken)
+ }
+
+ return r, nil
+}
+
+func userMatch() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("unknown user, please login first"))
+ ctx.Abort()
+ return
+ }
+
+ userName := ctx.Param("username")
+ if userName != currentUser {
+ httpbase.UnauthorizedError(ctx, errors.New("user not match, try to query user account not owned"))
+ slog.Error("user not match, try to query user account not owned", "currentUser", currentUser, "userName", userName)
+ ctx.Abort()
+ return
+ }
+ }
+}
+
+func mustLogin() gin.HandlerFunc {
+ return func(ctx *gin.Context) {
+ currentUser := httpbase.GetCurrentUser(ctx)
+ if currentUser == "" {
+ httpbase.UnauthorizedError(ctx, errors.New("unknown user, please login first"))
+ ctx.Abort()
+ return
+ }
+ }
+}
+
+
+
package activity
+
+import (
+ "context"
+ "fmt"
+
+ "go.temporal.io/sdk/activity"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/component"
+ "opencsg.com/csghub-server/user/workflow/common"
+)
+
+func DeleteUserAndRelations(ctx context.Context, user common.User, config *config.Config) error {
+ logger := activity.GetLogger(ctx)
+ logger.Info("delete user and relations start", "username", user.Username, "operator", user.Operator)
+ userComponent, err := component.NewUserComponent(config)
+ if err != nil {
+ return fmt.Errorf("failed to create user component, error: %w", err)
+ }
+ return userComponent.Delete(ctx, user.Operator, user.Username)
+}
+
+
+
package workflow
+
+import (
+ "time"
+
+ "go.temporal.io/sdk/temporal"
+ "go.temporal.io/sdk/workflow"
+
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/workflow/activity"
+ "opencsg.com/csghub-server/user/workflow/common"
+)
+
+func UserDeletionWorkflow(ctx workflow.Context, user common.User, config *config.Config) error {
+ logger := workflow.GetLogger(ctx)
+ logger.Info("user deletion workflow started")
+
+ retryPolicy := &temporal.RetryPolicy{
+ MaximumAttempts: 3,
+ }
+
+ options := workflow.ActivityOptions{
+ StartToCloseTimeout: time.Hour * 1,
+ RetryPolicy: retryPolicy,
+ }
+
+ ctx = workflow.WithActivityOptions(ctx, options)
+ err := workflow.ExecuteActivity(ctx, activity.DeleteUserAndRelations, user, config).Get(ctx, nil)
+ if err != nil {
+ logger.Error("failed to delete user and relations", "error", err, "user", user)
+ return err
+ }
+
+ return nil
+}
+
+
+
package workflow
+
+import (
+ "fmt"
+
+ "go.temporal.io/sdk/client"
+ "go.temporal.io/sdk/worker"
+ "opencsg.com/csghub-server/common/config"
+ "opencsg.com/csghub-server/user/workflow/activity"
+)
+
+const WorkflowUserDeletionQueueName = "workflow_user_deletion_queue"
+
+var wfWorker worker.Worker
+var wfClient client.Client
+
+func StartWorker(config *config.Config) error {
+ var err error
+ wfClient, err = client.Dial(client.Options{
+ HostPort: config.WorkFLow.Endpoint,
+ })
+ if err != nil {
+ return fmt.Errorf("unable to create workflow client, error:%w", err)
+ }
+ wfWorker = worker.New(wfClient, WorkflowUserDeletionQueueName, worker.Options{})
+ wfWorker.RegisterWorkflow(UserDeletionWorkflow)
+ wfWorker.RegisterActivity(activity.DeleteUserAndRelations)
+
+ return wfWorker.Start()
+}
+
+func StopWorker() {
+ if wfWorker != nil {
+ wfWorker.Stop()
+ }
+ if wfClient != nil {
+ wfClient.Close()
+ }
+}
+
+func GetWorkflowClient() client.Client {
+ return wfClient
+}
+
+
+