Mysql support

go.mod

@@ -5,19 +5,21 @@ go 1.19
 require (
 	github.com/Selvatico/go-mocket v1.0.7
 	github.com/flyteorg/flyteidl v1.3.6
-	github.com/flyteorg/flytestdlib v1.0.15
+	github.com/flyteorg/flytestdlib v1.0.17-0.20230320195919-90331d171e2a
+	github.com/go-gormigrate/gormigrate/v2 v2.0.2
+	github.com/go-sql-driver/mysql v1.7.0
 	github.com/gofrs/uuid v4.2.0+incompatible
 	github.com/golang/glog v1.0.0
 	github.com/golang/protobuf v1.5.2
-	github.com/jackc/pgconn v1.10.1
+	github.com/jackc/pgconn v1.13.0
 	github.com/mitchellh/mapstructure v1.4.3
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.0
 	google.golang.org/grpc v1.46.0
-	gorm.io/driver/postgres v1.2.3
-	gorm.io/driver/sqlite v1.1.1
-	gorm.io/gorm v1.22.4
+	gorm.io/driver/postgres v1.4.5
+	gorm.io/driver/sqlite v1.4.4
+	gorm.io/gorm v1.25.0
 )
 
 require (
@@ -56,17 +58,17 @@ require (
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgio v1.0.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.2.0 // indirect
+	github.com/jackc/pgproto3/v2 v2.3.1 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.9.0 // indirect
-	github.com/jackc/pgx/v4 v4.14.0 // indirect
+	github.com/jackc/pgtype v1.12.0 // indirect
+	github.com/jackc/pgx/v4 v4.17.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.4 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/mattn/go-sqlite3 v1.14.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/ncw/swift v1.0.53 // indirect
 	github.com/pelletier/go-toml v1.9.4 // indirect
@@ -82,10 +84,10 @@ require (
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.11.0 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/stretchr/objx v0.4.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
 	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
@@ -98,7 +100,8 @@ require (
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/mysql v1.5.0 // indirect
 	k8s.io/apimachinery v0.20.2 // indirect
 	k8s.io/client-go v0.0.0-20210217172142-7279fc64d847 // indirect
 	k8s.io/klog/v2 v2.5.0 // indirect

pkg/repositories/config/migrations.go

@@ -0,0 +1,19 @@
+package config
+
+import (
+	fixupmigrations "github.com/flyteorg/datacatalog/pkg/repositories/config/migrations/fixup"
+	noopmigrations "github.com/flyteorg/datacatalog/pkg/repositories/config/migrations/noop"
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+
+func ListMigrations(db *gorm.DB) []*gormigrate.Migration {
+	var Migrations []*gormigrate.Migration
+	if db.Dialector.Name() == "postgres" {
+		Migrations = append(Migrations, noopmigrations.Migrations...)
+	}
+	Migrations = append(Migrations, fixupmigrations.Migrations...)
+	return Migrations
+}
+

pkg/repositories/config/migrations/fixup/migrations.go

@@ -0,0 +1,194 @@
+package fixupmigrations
+
+import (
+	"time"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+	"gorm.io/gorm/schema"
+)
+
+type UUIDString string
+
+func (uuidString UUIDString) GormDBDataType(db *gorm.DB, field *schema.Field) string {
+	// use field.Tag, field.TagSettings gets field's tags
+	// checkout https://github.com/go-gorm/gorm/blob/master/schema/field.go for all options
+	if db.Dialector.Name() == "mysql" {
+		return "varchar(36)"
+	}
+	return "uuid"
+}
+
+type BaseModel struct {
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time `sql:"index"`
+}
+
+type ArtifactKey struct {
+	DatasetProject string `gorm:"size:64;primary_key"`
+	// Can we use a smaller size fo dataset name? This is `flyte-task-<task name>` 
+	DatasetName    string `gorm:"size:100;primary_key"`
+	DatasetDomain  string `gorm:"size:64;primary_key"`
+	DatasetVersion string `gorm:"size:128;primary_key"`
+	// This is a UUID
+	ArtifactID     string `gorm:"size:36;primary_key"`
+}
+
+type Artifact struct {
+	BaseModel
+	ArtifactKey
+	DatasetUUID        UUIDString     `gorm:"type:uuid;index:artifacts_dataset_uuid_idx"`
+	Dataset            Dataset        `gorm:"association_autocreate:false"`
+	ArtifactData       []ArtifactData `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"`
+	Partitions         []Partition    `gorm:"references:ArtifactID;foreignkey:ArtifactID"`
+	Tags               []Tag          `gorm:"references:ArtifactID,DatasetUUID;foreignkey:ArtifactID,DatasetUUID"`
+	SerializedMetadata []byte
+}
+
+type ArtifactData struct {
+	BaseModel
+	ArtifactKey
+	Name     string `gorm:"size:32;primary_key"`
+	Location string `gorm:"size:2048"`
+}
+
+
+type DatasetKey struct {
+	Project string     `gorm:"size:64;primary_key"`                          // part of pkey, no index needed as it is first column in the pkey
+	// TODO: figure out what size this should be
+	Name    string     `gorm:"size:100;primary_key;index:dataset_name_idx"`    // part of pkey and has separate index for filtering
+	Domain  string     `gorm:"size:64;primary_key;index:dataset_domain_idx"`  // part of pkey and has separate index for filtering
+	Version string     `gorm:"size:128;primary_key;index:dataset_version_idx"` // part of pkey and has separate index for filtering
+	UUID    UUIDString `gorm:"unique;type:uuid"`
+}
+
+type Dataset struct {
+	BaseModel
+	DatasetKey
+	SerializedMetadata []byte
+	PartitionKeys      []PartitionKey `gorm:"references:UUID;foreignkey:DatasetUUID"`
+}
+
+type PartitionKey struct {
+	BaseModel
+	DatasetUUID UUIDString `gorm:"type:uuid;primary_key"`
+	// TODO: figure out if this is used.
+	Name        string `gorm:"size:100;primary_key"`
+}
+
+type TagKey struct {
+	DatasetProject string `gorm:"size:64;primary_key"`
+	// TODO: figure out what size this should be
+	DatasetName    string `gorm:"size:100;primary_key"`
+	DatasetDomain  string `gorm:"size:64;primary_key"`
+	DatasetVersion string `gorm:"size:128;primary_key"`
+	TagName        string `gorm:"size:56;primary_key"`
+}
+
+type Tag struct {
+	BaseModel
+	TagKey
+	ArtifactID  string     `gorm:"size:36"`
+	DatasetUUID UUIDString `gorm:"index:tags_dataset_uuid_idx"`
+	Artifact    Artifact   `gorm:"references:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID;foreignkey:DatasetProject,DatasetName,DatasetDomain,DatasetVersion,ArtifactID"`
+}
+
+// TODO: figure out if this is used.
+type Partition struct {
+	BaseModel
+	DatasetUUID UUIDString `gorm:"primary_key;type:uuid"`
+	Key         string `gorm:"primary_key"`
+	Value       string `gorm:"primary_key"`
+	ArtifactID  string `gorm:"primary_key;index"` // index for JOINs with the Tag/Labels table when querying artifacts
+}
+
+type ReservationKey struct {
+	DatasetProject string `gorm:"size:64;primary_key"`
+	// TODO: figure out what size this should be
+	DatasetName    string `gorm:"size:100;primary_key"`
+	DatasetDomain  string `gorm:"size:64;primary_key"`
+	DatasetVersion string `gorm:"size:128;primary_key"`
+	// TODO: figure out what size this should be
+	TagName        string `gorm:"56;primary_key"`
+}
+
+// Reservation tracks the metadata needed to allow
+// task cache serialization
+type Reservation struct {
+	BaseModel
+	ReservationKey
+
+	// Identifies who owns the reservation
+	OwnerID string
+
+	// When the reservation will expire
+	ExpiresAt          time.Time
+	SerializedMetadata []byte
+}
+
+var Migrations = []*gormigrate.Migration{
+	{
+		ID: "2023-04-18-fixup-dataset",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&Dataset{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+	{
+		ID: "2023-04-18-fixup-artifact",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&Artifact{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+	{
+		ID: "2023-04-18-fixup-artifact-data",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&ArtifactData{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+	{
+		ID: "2023-04-18-fixup-tag",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&Tag{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+	{
+		ID: "2023-04-18-fixup-partition-key",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&PartitionKey{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+	{
+		ID: "2023-04-18-fixup-partition",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&Partition{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+	{
+		ID: "2023-04-18-fixup-reservation",
+		Migrate: func(tx *gorm.DB) error {
+			return tx.AutoMigrate(&Reservation{})
+		},
+		Rollback: func(tx *gorm.DB) error {
+			return nil
+		},
+	},
+}