-
Notifications
You must be signed in to change notification settings - Fork 673
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Yee Hing Tong <[email protected]>
- Loading branch information
1 parent
e830db5
commit 644d2d1
Showing
19 changed files
with
534 additions
and
187 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,3 +3,6 @@ artifactsServer: | |
| database: | ||
| postgres: | ||
| dbname: artifacts | ||
| logger: | ||
| level: 5 | ||
| show-source: true | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| type ArtifactKey struct { | ||
| gorm.Model | ||
| Project string `gorm:"index:idx_pdn;index:idx_proj;type:varchar(64)"` | ||
| Domain string `gorm:"index:idx_pdn;index:idx_dom;type:varchar(64)"` | ||
| Name string `gorm:"index:idx_pdn;index:idx_name;type:varchar(255)"` | ||
| } | ||
| type Artifact struct { | ||
| gorm.Model | ||
| // gatepr: this doesn't actually create a foreign key... | ||
| ArtifactKeyID uint | ||
| ArtifactKey ArtifactKey `gorm:"foreignKey:ArtifactKeyID;references:ID"` | ||
| Version string `gorm:"not null;type:varchar(255);index:idx_artifact_version"` | ||
| ExecutionName string `gorm:"type:varchar(255)"` | ||
| } | ||
|
|
||
| //type Artifact struct { | ||
| // gorm.Model | ||
| // // gatepr: this doesn't actually create a foreign key... | ||
| // ArtifactKeyID uint | ||
| // ArtifactKey ArtifactKey `gorm:"foreignKey:ArtifactKeyID;references:ID"` | ||
| // Version string `gorm:"not null;type:varchar(255);index:idx_artifact_version"` | ||
| // Partitions postgres.Hstore `gorm:"type:hstore;index:idx_artifact_partitions"` | ||
| // | ||
| // LiteralType []byte `gorm:"not null"` | ||
| // LiteralValue []byte `gorm:"not null"` | ||
| // | ||
| // Description string `gorm:"type:varchar(255)"` | ||
| // MetadataType string `gorm:"type:varchar(64)"` | ||
| // OffloadedUserMetadata string `gorm:"type:varchar(255)"` | ||
| // | ||
| // // Project/Domain assumed to always be the same as the Artifact | ||
| // ExecutionName string `gorm:"type:varchar(255)"` | ||
| // WorkflowProject string `gorm:"type:varchar(64)"` | ||
| // WorkflowDomain string `gorm:"type:varchar(64)"` | ||
| // WorkflowName string `gorm:"type:varchar(255)"` | ||
| // WorkflowVersion string `gorm:"type:varchar(255)"` | ||
| // TaskProject string `gorm:"type:varchar(64)"` | ||
| // TaskDomain string `gorm:"type:varchar(64)"` | ||
| // TaskName string `gorm:"type:varchar(255)"` | ||
| // TaskVersion string `gorm:"type:varchar(255)"` | ||
| // NodeID string `gorm:"type:varchar(64)"` | ||
| // // See Admin migration for note. | ||
| // // Here nullable in the case of workflow output. | ||
| // RetryAttempt *uint32 | ||
| //} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/flyteorg/flyte/flytestdlib/promutils" | ||
| ) | ||
|
|
||
| // Common metrics emitted by gormimpl repos. | ||
| type gormMetrics struct { | ||
| Scope promutils.Scope | ||
| CreateDuration promutils.StopWatch | ||
| GetDuration promutils.StopWatch | ||
| UpdateDuration promutils.StopWatch | ||
| SearchDuration promutils.StopWatch | ||
| } | ||
|
|
||
| func newMetrics(scope promutils.Scope) gormMetrics { | ||
| return gormMetrics{ | ||
| Scope: scope, | ||
| CreateDuration: scope.MustNewStopWatch( | ||
| "create", "time taken to create a new entry", time.Millisecond), | ||
| GetDuration: scope.MustNewStopWatch( | ||
| "get", "time taken to get an entry", time.Millisecond), | ||
| UpdateDuration: scope.MustNewStopWatch( | ||
| "update", "time taken to update an entry", time.Millisecond), | ||
| SearchDuration: scope.MustNewStopWatch( | ||
| "search", "time taken for searching", time.Millisecond), | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "github.com/go-gormigrate/gormigrate/v2" | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| var Migrations = []*gormigrate.Migration{ | ||
| { | ||
| ID: "2023-10-12-inits", | ||
| Migrate: func(tx *gorm.DB) error { | ||
| type ArtifactKey struct { | ||
| gorm.Model | ||
| Project string `gorm:"uniqueIndex:idx_pdn;index:idx_proj;type:varchar(64)"` | ||
| Domain string `gorm:"uniqueIndex:idx_pdn;index:idx_dom;type:varchar(64)"` | ||
| Name string `gorm:"uniqueIndex:idx_pdn;index:idx_name;type:varchar(255)"` | ||
| } | ||
| type Artifact struct { | ||
| gorm.Model | ||
| ArtifactKeyID uint `gorm:"uniqueIndex:idx_pdnv"` | ||
| ArtifactKey ArtifactKey `gorm:"foreignKey:ArtifactKeyID;references:ID"` | ||
| Version string `gorm:"type:varchar(255);index:idx_artifact_version;uniqueIndex:idx_pdnv"` | ||
| ExecutionName string `gorm:"type:varchar(255)"` | ||
| } | ||
|
|
||
| err := tx.AutoMigrate( | ||
| &ArtifactKey{}, &Artifact{}, | ||
| ) | ||
| return err | ||
| }, | ||
| Rollback: func(tx *gorm.DB) error { | ||
| return tx.Migrator().DropTable( | ||
| "artifact_keys", "artifacts", | ||
| ) | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| //var Migrations = []*gormigrate.Migration{ | ||
| // { | ||
| // ID: "2023-10-12-inits", | ||
| // Migrate: func(tx *gorm.DB) error { | ||
| // type ArtifactKey struct { | ||
| // gorm.Model | ||
| // Project string `gorm:"index:idx_pdn;index:idx_proj;type:varchar(64)"` | ||
| // Domain string `gorm:"index:idx_pdn;index:idx_dom;type:varchar(64)"` | ||
| // Name string `gorm:"index:idx_pdn;index:idx_name;type:varchar(255)"` | ||
| // } | ||
| // type Artifact struct { | ||
| // gorm.Model | ||
| // ArtifactKeyID uint `gorm:"uniqueIndex:idx_pdnv"` | ||
| // ArtifactKey ArtifactKey `gorm:"foreignKey:ArtifactKeyID;references:ID"` | ||
| // Version string `gorm:"type:varchar(255);index:idx_artifact_version;uniqueIndex:idx_pdnv"` | ||
| // Partitions postgres.Hstore `gorm:"type:hstore;index:idx_artifact_partitions"` | ||
| // | ||
| // LiteralType []byte `gorm:"not null"` | ||
| // LiteralValue []byte `gorm:"not null"` | ||
| // | ||
| // Description string `gorm:"type:varchar(255)"` | ||
| // MetadataType string `gorm:"type:varchar(64)"` | ||
| // OffloadedUserMetadata string `gorm:"type:varchar(255)"` | ||
| // | ||
| // // Project/Domain assumed to always be the same as the Artifact | ||
| // ExecutionName string `gorm:"type:varchar(255)"` | ||
| // WorkflowProject string `gorm:"type:varchar(64)"` | ||
| // WorkflowDomain string `gorm:"type:varchar(64)"` | ||
| // WorkflowName string `gorm:"type:varchar(255)"` | ||
| // WorkflowVersion string `gorm:"type:varchar(255)"` | ||
| // TaskProject string `gorm:"type:varchar(64)"` | ||
| // TaskDomain string `gorm:"type:varchar(64)"` | ||
| // TaskName string `gorm:"type:varchar(255)"` | ||
| // TaskVersion string `gorm:"type:varchar(255)"` | ||
| // NodeID string `gorm:"type:varchar(64)"` | ||
| // // See Admin migration for note. | ||
| // // Here nullable in the case of workflow output. | ||
| // RetryAttempt *uint32 | ||
| // } | ||
| // return tx.AutoMigrate( | ||
| // &ArtifactKey{}, &Artifact{}, | ||
| // ) | ||
| // }, | ||
| // Rollback: func(tx *gorm.DB) error { | ||
| // return tx.Migrator().DropTable( | ||
| // "markers", | ||
| // ) | ||
| // }, | ||
| // }, | ||
| //} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "context" | ||
| "github.com/flyteorg/flyte/flyteartifacts/pkg/models" | ||
| "github.com/flyteorg/flyte/flytestdlib/database" | ||
| "github.com/flyteorg/flyte/flytestdlib/logger" | ||
| "github.com/flyteorg/flyte/flytestdlib/promutils" | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| // RDSStorage should implement StorageInterface | ||
| type RDSStorage struct { | ||
| config database.DbConfig | ||
| db *gorm.DB | ||
| metrics gormMetrics | ||
| } | ||
|
|
||
| // WriteOne is a test function | ||
| func (r *RDSStorage) WriteOne(ctx context.Context, gormModel Artifact) (models.Artifact, error) { | ||
| timer := r.metrics.CreateDuration.Start() | ||
| logger.Debugf(ctx, "Attempt create artifact %s", gormModel.Version) | ||
| tx := r.db.Omit("id").Create(&gormModel) | ||
| timer.Stop() | ||
| if tx.Error != nil { | ||
| return models.Artifact{}, tx.Error | ||
| } | ||
| return models.Artifact{}, nil | ||
| } | ||
|
|
||
| func NewStorage(ctx context.Context, scope promutils.Scope) *RDSStorage { | ||
| dbCfg := database.GetConfig() | ||
| logConfig := logger.GetConfig() | ||
|
|
||
| db, err := database.GetDB(ctx, dbCfg, logConfig) | ||
| if err != nil { | ||
| logger.Fatal(ctx, err) | ||
| } | ||
| return &RDSStorage{ | ||
| config: *dbCfg, | ||
| db: db, | ||
| metrics: newMetrics(scope.NewSubScope("rds")), | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| //go:build local_integration | ||
|
|
||
| package db | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/flyteorg/flyte/flytestdlib/config" | ||
| "github.com/flyteorg/flyte/flytestdlib/config/viper" | ||
| "github.com/flyteorg/flyte/flytestdlib/promutils" | ||
| "github.com/stretchr/testify/assert" | ||
| "testing" | ||
| ) | ||
|
|
||
| func TestWriteOne(t *testing.T) { | ||
| ctx := context.Background() | ||
| configAccessor := viper.NewAccessor(config.Options{ | ||
| SearchPaths: []string{"/Users/ytong/go/src/github.com/flyteorg/flyte/flyteartifacts/sandbox.yaml"}, | ||
| StrictMode: false, | ||
| }) | ||
| err := configAccessor.UpdateConfig(ctx) | ||
|
|
||
| fmt.Println("Local integration testing using: ", configAccessor.ConfigFilesUsed()) | ||
| scope := promutils.NewTestScope() | ||
| rds := NewStorage(ctx, scope) | ||
|
|
||
| gormA := Artifact{ | ||
| ArtifactKey: ArtifactKey{ | ||
| Project: "demotst", | ||
| Domain: "unit", | ||
| Name: "testname 1", | ||
| }, | ||
| Version: "abc123/1/n0/4", | ||
| ExecutionName: "ddd", | ||
| } | ||
|
|
||
| a, err := rds.WriteOne(ctx, gormA) | ||
| assert.NoError(t, err) | ||
| fmt.Println(a, err) | ||
| } | ||
|
|
||
| //func TestWriteOne(t *testing.T) { | ||
| // ctx := context.Background() | ||
| // configAccessor := viper.NewAccessor(config.Options{ | ||
| // SearchPaths: []string{"/Users/ytong/go/src/github.com/flyteorg/flyte/flyteartifacts/sandbox.yaml"}, | ||
| // StrictMode: false, | ||
| // }) | ||
| // err := configAccessor.UpdateConfig(ctx) | ||
| // | ||
| // fmt.Println("Local integration testing using: ", configAccessor.ConfigFilesUsed()) | ||
| // scope := promutils.NewTestScope() | ||
| // rds := NewStorage(ctx, scope) | ||
| // | ||
| // one := uint32(1) | ||
| // pval1 := "51" | ||
| // p := map[string]*string{"area": &pval1} | ||
| // | ||
| // lt := &core.LiteralType{ | ||
| // Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}, | ||
| // } | ||
| // lit := &core.Literal{ | ||
| // Value: &core.Literal_Scalar{ | ||
| // Scalar: &core.Scalar{ | ||
| // Value: &core.Scalar_Primitive{ | ||
| // &core.Primitive{ | ||
| // Value: &core.Primitive_Integer{15}, | ||
| // }, | ||
| // }, | ||
| // }, | ||
| // }, | ||
| // } | ||
| // | ||
| // ltBytes, err := proto.Marshal(lt) | ||
| // assert.NoError(t, err) | ||
| // litBytes, err := proto.Marshal(lit) | ||
| // assert.NoError(t, err) | ||
| // | ||
| // gormA := Artifact{ | ||
| // ArtifactKey: ArtifactKey{ | ||
| // Project: "demotst", | ||
| // Domain: "unit", | ||
| // Name: "testname 1", | ||
| // }, | ||
| // Version: "abc123/1/n0/4", | ||
| // ExecutionName: "ddd", | ||
| // } | ||
| // | ||
| // a, err := rds.WriteOne(ctx, gormA) | ||
| // fmt.Println(a, err) | ||
| //} |
Oops, something went wrong.