From bb00fef68551cb17a432aa3e413f88cfb2994db7 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Tue, 17 Dec 2024 23:58:36 -0800 Subject: [PATCH 01/23] Jai/hyp 2765 get api (#28) --- api.go | 386 +++++++++++++++++++++++++++++++++++++++++++++++++++ api_test.go | 103 ++++++++++++++ go.mod | 6 +- go.sum | 12 +- namespace.go | 15 +- zero.go | 8 ++ 6 files changed, 518 insertions(+), 12 deletions(-) create mode 100644 api.go create mode 100644 api_test.go diff --git a/api.go b/api.go new file mode 100644 index 0000000..d5c5341 --- /dev/null +++ b/api.go @@ -0,0 +1,386 @@ +package modusdb + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "reflect" + "strings" + "time" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/dql" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/query" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/worker" + "github.com/dgraph-io/dgraph/v24/x" + "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/wkb" +) + +type UniqueField interface { + uint64 | ConstrainedField +} +type ConstrainedField struct { + Key string + Value any +} + +func getFieldTags(t reflect.Type) (jsonTags map[string]string, reverseEdgeTags map[string]string, err error) { + jsonTags = make(map[string]string) + reverseEdgeTags = make(map[string]string) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + return nil, nil, fmt.Errorf("field %s has no json tag", field.Name) + } + jsonName := strings.Split(jsonTag, ",")[0] + jsonTags[field.Name] = jsonName + reverseEdgeTag := field.Tag.Get("readFrom") + if reverseEdgeTag != "" { + typeAndField := strings.Split(reverseEdgeTag, ",") + if len(typeAndField) != 2 { + return nil, nil, fmt.Errorf(`field %s has invalid readFrom tag, + expected format is type=,field=`, field.Name) + } + t := strings.Split(typeAndField[0], "=")[1] + f := strings.Split(typeAndField[1], "=")[1] + reverseEdgeTags[field.Name] = getPredicateName(t, f) + } + } + return jsonTags, reverseEdgeTags, nil +} + +func getFieldValues(object any, jsonFields map[string]string) map[string]any { + values := make(map[string]any) + v := reflect.ValueOf(object).Elem() + for fieldName, jsonName := range jsonFields { + fieldValue := v.FieldByName(fieldName) + values[jsonName] = fieldValue.Interface() + + } + return values +} + +func getPredicateName(typeName, fieldName string) string { + return fmt.Sprint(typeName, ".", fieldName) +} + +func addNamespace(ns uint64, pred string) string { + return x.NamespaceAttr(ns, pred) +} + +func valueToPosting_ValType(v any) (pb.Posting_ValType, error) { + switch v.(type) { + case string: + return pb.Posting_STRING, nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return pb.Posting_INT, nil + case bool: + return pb.Posting_BOOL, nil + case float32, float64: + return pb.Posting_FLOAT, nil + case []byte: + return pb.Posting_BINARY, nil + case time.Time: + return pb.Posting_DATETIME, nil + case geom.Point: + return pb.Posting_GEO, nil + case []float32, []float64: + return pb.Posting_VFLOAT, nil + default: + return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) + } +} + +func valueToValType(v any) (*api.Value, error) { + switch val := v.(type) { + case string: + return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil + case int: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int64: + return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil + case uint8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case bool: + return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil + case float32: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil + case float64: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil + case []byte: + return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil + case time.Time: + bytes, err := val.MarshalBinary() + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil + case geom.Point: + bytes, err := wkb.Marshal(&val, binary.LittleEndian) + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil + case uint, uint64: + return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil + default: + return nil, fmt.Errorf("unsupported type %T", v) + } +} + +func generateDqlMutationAndSchema[T any](n *Namespace, object *T, + uid uint64) ([]*dql.Mutation, *schema.ParsedSchema, error) { + t := reflect.TypeOf(*object) + if t.Kind() != reflect.Struct { + return nil, nil, fmt.Errorf("expected struct, got %s", t.Kind()) + } + + jsonFields, _, err := getFieldTags(t) + if err != nil { + return nil, nil, err + } + values := getFieldValues(object, jsonFields) + sch := &schema.ParsedSchema{} + + nquads := make([]*api.NQuad, 0) + for jsonName, value := range values { + if jsonName == "uid" { + continue + } + valType, err := valueToPosting_ValType(value) + if err != nil { + return nil, nil, err + } + sch.Preds = append(sch.Preds, &pb.SchemaUpdate{ + Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), + ValueType: valType, + }) + val, err := valueToValType(value) + if err != nil { + return nil, nil, err + } + nquad := &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(uid), + Predicate: getPredicateName(t.Name(), jsonName), + ObjectValue: val, + } + nquads = append(nquads, nquad) + } + sch.Types = append(sch.Types, &pb.TypeUpdate{ + TypeName: addNamespace(n.id, t.Name()), + Fields: sch.Preds, + }) + + val, err := valueToValType(t.Name()) + if err != nil { + return nil, nil, err + } + nquad := &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(uid), + Predicate: "dgraph.type", + ObjectValue: val, + } + nquads = append(nquads, nquad) + + dms := make([]*dql.Mutation, 0) + dms = append(dms, &dql.Mutation{ + Set: nquads, + }) + + return dms, sch, nil +} + +func Create[T any](ctx context.Context, n *Namespace, object *T) (uint64, *T, error) { + n.db.mutex.Lock() + defer n.db.mutex.Unlock() + uid, err := n.db.z.nextUID() + if err != nil { + return 0, object, err + } + + dms, sch, err := generateDqlMutationAndSchema(n, object, uid) + if err != nil { + return 0, object, err + } + + edges, err := query.ToDirectedEdges(dms, nil) + if err != nil { + return 0, object, err + } + ctx = x.AttachNamespace(ctx, n.ID()) + + err = n.alterSchemaWithParsed(ctx, sch) + if err != nil { + return 0, object, err + } + + if !n.db.isOpen { + return 0, object, ErrClosedDB + } + + startTs, err := n.db.z.nextTs() + if err != nil { + return 0, object, err + } + commitTs, err := n.db.z.nextTs() + if err != nil { + return 0, object, err + } + + m := &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Edges: edges, + } + m.Edges, err = query.ExpandEdges(ctx, m) + if err != nil { + return 0, object, fmt.Errorf("error expanding edges: %w", err) + } + + p := &pb.Proposal{Mutations: m, StartTs: startTs} + if err := worker.ApplyMutations(ctx, p); err != nil { + return 0, object, err + } + + err = worker.ApplyCommited(ctx, &pb.OracleDelta{ + Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, + }) + if err != nil { + return 0, object, err + } + + v := reflect.ValueOf(object).Elem() + + uidField := v.FieldByName("Uid") + + if uidField.IsValid() && uidField.CanSet() && uidField.Kind() == reflect.Uint64 { + uidField.SetUint(uid) + } + + return uid, object, nil +} + +func createDynamicStruct(t reflect.Type, jsonFields map[string]string) reflect.Type { + fields := make([]reflect.StructField, 0, len(jsonFields)) + for fieldName, jsonName := range jsonFields { + field, _ := t.FieldByName(fieldName) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: field.Type, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } + return reflect.StructOf(fields) +} + +func mapDynamicToFinal(dynamic any, final any) { + vFinal := reflect.ValueOf(final).Elem() + vDynamic := reflect.ValueOf(dynamic).Elem() + + for i := 0; i < vDynamic.NumField(); i++ { + field := vDynamic.Type().Field(i) + value := vDynamic.Field(i) + + finalField := vFinal.FieldByName(field.Name) + if finalField.IsValid() && finalField.CanSet() { + finalField.Set(value) + } + } +} + +func Get[T any, R UniqueField](ctx context.Context, n *Namespace, uniqueField R) (*T, error) { + if uid, ok := any(uniqueField).(uint64); ok { + return getByUid[T](ctx, n, uid) + } + + if cf, ok := any(uniqueField).(ConstrainedField); ok { + return getByConstrainedField[T](ctx, n, cf) + } + + return nil, fmt.Errorf("invalid unique field type") +} + +func getByUid[T any](ctx context.Context, n *Namespace, uid uint64) (*T, error) { + query := fmt.Sprintf(` + { + obj(func: uid(%d)) { + uid + expand(_all_) + } + } + `, uid) + + return executeGet[T](ctx, n, query) +} + +func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (*T, error) { + query := fmt.Sprintf(` + { + obj(func: eq(%s, %s)) { + uid + expand(_all_) + } + } + `, cf.Key, cf.Value) + + return executeGet[T](ctx, n, query) +} + +func executeGet[T any](ctx context.Context, n *Namespace, query string) (*T, error) { + resp, err := n.Query(ctx, query) + if err != nil { + return nil, err + } + + var obj T + + t := reflect.TypeOf(obj) + + jsonFields, _, err := getFieldTags(t) + if err != nil { + return nil, err + } + + dynamicType := createDynamicStruct(t, jsonFields) + + dynamicInstance := reflect.New(dynamicType).Interface() + + var result struct { + Obj []any `json:"obj"` + } + + result.Obj = append(result.Obj, dynamicInstance) + + // Unmarshal the JSON response into the dynamic struct + if err := json.Unmarshal(resp.Json, &result); err != nil { + return nil, err + } + + // Check if we have at least one object in the response + if len(result.Obj) == 0 { + return nil, fmt.Errorf("no object found") + } + + // Map the dynamic struct to the final type T + finalObject := reflect.New(t).Interface() + mapDynamicToFinal(result.Obj[0], finalObject) + + return finalObject.(*T), nil +} diff --git a/api_test.go b/api_test.go new file mode 100644 index 0000000..ff0cb45 --- /dev/null +++ b/api_test.go @@ -0,0 +1,103 @@ +package modusdb_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/hypermodeinc/modusdb" +) + +type User struct { + Uid uint64 `json:"uid,omitempty"` + Name string `json:"name,omitempty"` + Age int `json:"age,omitempty"` +} + +func TestCreateApi(t *testing.T) { + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(context.Background())) + + user := &User{ + Name: "B", + Age: 20, + } + + uid, _, err := modusdb.Create(context.Background(), db1, user) + require.NoError(t, err) + + require.Equal(t, "B", user.Name) + require.Equal(t, uint64(2), uid) + require.Equal(t, uint64(2), user.Uid) + + query := `{ + me(func: has(User.name)) { + uid + User.name + User.age + } + }` + resp, err := db1.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[{"uid":"0x2","User.name":"B","User.age":20}]}`, string(resp.GetJson())) + + // TODO schema{} should work + resp, err = db1.Query(context.Background(), `schema(pred: [User.name, User.age]) {type}`) + require.NoError(t, err) + + require.JSONEq(t, + `{"schema":[{"predicate":"User.age","type":"int"},{"predicate":"User.name","type":"string"}]}`, + string(resp.GetJson())) +} + +func TestCreateApiWithNonStruct(t *testing.T) { + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(context.Background())) + + user := &User{ + Name: "B", + Age: 20, + } + + _, _, err = modusdb.Create[*User](context.Background(), db1, &user) + require.Error(t, err) + require.Equal(t, "expected struct, got ptr", err.Error()) +} + +func TestGetApi(t *testing.T) { + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(context.Background())) + + user := &User{ + Name: "B", + Age: 20, + } + + _, _, err = modusdb.Create(context.Background(), db1, user) + require.NoError(t, err) + + userQuery, err := modusdb.Get[User](context.Background(), db1, uint64(2)) + + require.NoError(t, err) + require.Equal(t, 20, userQuery.Age) + require.Equal(t, "B", userQuery.Name) +} diff --git a/go.mod b/go.mod index 7253865..291d7c4 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/dgraph-io/ristretto/v2 v2.0.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.10.0 - golang.org/x/sync v0.9.0 + golang.org/x/sync v0.10.0 ) require ( @@ -124,9 +124,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.29.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect golang.org/x/net v0.31.0 // indirect - golang.org/x/sys v0.27.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.26.0 // indirect golang.org/x/text v0.20.0 // indirect golang.org/x/time v0.6.0 // indirect diff --git a/go.sum b/go.sum index 46296ee..152dd35 100644 --- a/go.sum +++ b/go.sum @@ -697,8 +697,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -787,8 +787,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -840,8 +840,8 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= diff --git a/namespace.go b/namespace.go index d9446df..9668804 100644 --- a/namespace.go +++ b/namespace.go @@ -61,6 +61,10 @@ func (n *Namespace) AlterSchema(ctx context.Context, sch string) error { if err != nil { return fmt.Errorf("error parsing schema: %w", err) } + return n.alterSchemaWithParsed(ctx, sc) +} + +func (n *Namespace) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { for _, pred := range sc.Preds { worker.InitTablet(pred.Predicate) } @@ -87,6 +91,8 @@ func (n *Namespace) Mutate(ctx context.Context, ms []*api.Mutation) (map[string] return nil, nil } + n.db.mutex.Lock() + defer n.db.mutex.Unlock() dms := make([]*dql.Mutation, 0, len(ms)) for _, mu := range ms { dm, err := edgraph.ParseMutationObject(mu, false) @@ -113,15 +119,18 @@ func (n *Namespace) Mutate(ctx context.Context, ms []*api.Mutation) (map[string] curId++ } } + + return n.mutateWithDqlMutation(ctx, dms, newUids) +} + +func (n *Namespace) mutateWithDqlMutation(ctx context.Context, dms []*dql.Mutation, + newUids map[string]uint64) (map[string]uint64, error) { edges, err := query.ToDirectedEdges(dms, newUids) if err != nil { return nil, err } ctx = x.AttachNamespace(ctx, n.ID()) - n.db.mutex.Lock() - defer n.db.mutex.Unlock() - if !n.db.isOpen { return nil, ErrClosedDB } diff --git a/zero.go b/zero.go index 4aeb362..786724a 100644 --- a/zero.go +++ b/zero.go @@ -90,6 +90,14 @@ func (z *zero) readTs() uint64 { return z.minLeasedTs - 1 } +func (z *zero) nextUID() (uint64, error) { + uids, err := z.nextUIDs(&pb.Num{Val: 1, Type: pb.Num_UID}) + if err != nil { + return 0, err + } + return uids.StartId, nil +} + func (z *zero) nextUIDs(num *pb.Num) (pb.AssignedIds, error) { var resp pb.AssignedIds if num.Bump { From 47fec90280a8c5dc6c3b917195cf01ddc3019aaf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Dec 2024 08:30:20 -0500 Subject: [PATCH 02/23] Bump golang.org/x/crypto from 0.29.0 to 0.31.0 (#29) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.29.0 to 0.31.0.
Commits
  • b4f1988 ssh: make the public key cache a 1-entry FIFO cache
  • 7042ebc openpgp/clearsign: just use rand.Reader in tests
  • 3e90321 go.mod: update golang.org/x dependencies
  • 8c4e668 x509roots/fallback: update bundle
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.29.0&new-version=0.31.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/hypermodeinc/modusDB/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 291d7c4..8966a93 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/dgraph-io/ristretto/v2 v2.0.0 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.10.0 + github.com/twpayne/go-geom v1.5.7 golang.org/x/sync v0.10.0 ) @@ -113,7 +114,6 @@ require ( github.com/spf13/viper v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tinylib/msgp v1.2.1 // indirect - github.com/twpayne/go-geom v1.5.7 // indirect github.com/uber/jaeger-client-go v2.28.0+incompatible // indirect github.com/viterin/partial v1.1.0 // indirect github.com/viterin/vek v0.4.2 // indirect @@ -123,12 +123,12 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.29.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect golang.org/x/net v0.31.0 // indirect golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.26.0 // indirect - golang.org/x/text v0.20.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/api v0.196.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/go.sum b/go.sum index 152dd35..1f71c4e 100644 --- a/go.sum +++ b/go.sum @@ -682,8 +682,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -845,8 +845,8 @@ golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -855,8 +855,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 208821a33eb058fcdccd96652594369a2f6f5e1e Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Wed, 18 Dec 2024 07:29:16 -0800 Subject: [PATCH 03/23] separate ci tests and lint (#30) --- .github/workflows/ci-go-lint.yaml | 34 ++++++++++++++++++++++++++++++ .github/workflows/ci-go-tests.yaml | 5 ----- 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci-go-lint.yaml diff --git a/.github/workflows/ci-go-lint.yaml b/.github/workflows/ci-go-lint.yaml new file mode 100644 index 0000000..c9c3ddd --- /dev/null +++ b/.github/workflows/ci-go-lint.yaml @@ -0,0 +1,34 @@ +name: ci-go-tests + +on: + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + paths: + - '**/*.go' + - '**/go.mod' + +permissions: + contents: read + actions: write + +jobs: + ci-go-tests: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache-dependency-path: go.sum + + - name: golangci-lint + uses: golangci/golangci-lint-action@v6.1.1 + with: + args: --timeout=10m diff --git a/.github/workflows/ci-go-tests.yaml b/.github/workflows/ci-go-tests.yaml index 6fbcc81..579348c 100644 --- a/.github/workflows/ci-go-tests.yaml +++ b/.github/workflows/ci-go-tests.yaml @@ -28,10 +28,5 @@ jobs: go-version-file: 'go.mod' cache-dependency-path: go.sum - - name: golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 - with: - args: --timeout=10m - - name: Run Unit Tests run: go test -race -v ./... From 571b79961a01f6ee35892a91e50a9bd4dd08b558 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:03:44 -0500 Subject: [PATCH 04/23] Create CODEOWNERS --- .github/CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..cb93fb5 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# CODEOWNERS info: https://help.github.com/en/articles/about-code-owners +# Owners are automatically requested for review for PRs that changes code +# that they own. + +* @hypermodeinc/database From 0420655b6b6c11e1bd4f6e17770e8b1b58bb6df6 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Wed, 18 Dec 2024 17:38:15 -0800 Subject: [PATCH 05/23] add readFrom, index creation, and gid for apis (#31) --- .github/workflows/ci-go-lint.yaml | 4 +- api.go | 328 ++++-------------------------- api_helper.go | 237 +++++++++++++++++++++ api_reflect.go | 115 +++++++++++ api_test.go | 61 ++++-- api_types.go | 9 + db.go | 20 +- 7 files changed, 453 insertions(+), 321 deletions(-) create mode 100644 api_helper.go create mode 100644 api_reflect.go create mode 100644 api_types.go diff --git a/.github/workflows/ci-go-lint.yaml b/.github/workflows/ci-go-lint.yaml index c9c3ddd..e5bcfd8 100644 --- a/.github/workflows/ci-go-lint.yaml +++ b/.github/workflows/ci-go-lint.yaml @@ -1,4 +1,4 @@ -name: ci-go-tests +name: ci-go-lint on: pull_request: @@ -16,7 +16,7 @@ permissions: actions: write jobs: - ci-go-tests: + ci-go-lint: runs-on: ubuntu-24.04 steps: diff --git a/api.go b/api.go index d5c5341..db07298 100644 --- a/api.go +++ b/api.go @@ -2,220 +2,57 @@ package modusdb import ( "context" - "encoding/binary" - "encoding/json" "fmt" "reflect" - "strings" - "time" - "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/query" - "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/worker" "github.com/dgraph-io/dgraph/v24/x" - "github.com/twpayne/go-geom" - "github.com/twpayne/go-geom/encoding/wkb" ) -type UniqueField interface { - uint64 | ConstrainedField -} -type ConstrainedField struct { - Key string - Value any -} +type ModusDbOption func(*modusDbOptions) -func getFieldTags(t reflect.Type) (jsonTags map[string]string, reverseEdgeTags map[string]string, err error) { - jsonTags = make(map[string]string) - reverseEdgeTags = make(map[string]string) - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - jsonTag := field.Tag.Get("json") - if jsonTag == "" { - return nil, nil, fmt.Errorf("field %s has no json tag", field.Name) - } - jsonName := strings.Split(jsonTag, ",")[0] - jsonTags[field.Name] = jsonName - reverseEdgeTag := field.Tag.Get("readFrom") - if reverseEdgeTag != "" { - typeAndField := strings.Split(reverseEdgeTag, ",") - if len(typeAndField) != 2 { - return nil, nil, fmt.Errorf(`field %s has invalid readFrom tag, - expected format is type=,field=`, field.Name) - } - t := strings.Split(typeAndField[0], "=")[1] - f := strings.Split(typeAndField[1], "=")[1] - reverseEdgeTags[field.Name] = getPredicateName(t, f) - } - } - return jsonTags, reverseEdgeTags, nil +type modusDbOptions struct { + namespace uint64 } -func getFieldValues(object any, jsonFields map[string]string) map[string]any { - values := make(map[string]any) - v := reflect.ValueOf(object).Elem() - for fieldName, jsonName := range jsonFields { - fieldValue := v.FieldByName(fieldName) - values[jsonName] = fieldValue.Interface() - +func WithNamespace(namespace uint64) ModusDbOption { + return func(o *modusDbOptions) { + o.namespace = namespace } - return values -} - -func getPredicateName(typeName, fieldName string) string { - return fmt.Sprint(typeName, ".", fieldName) } -func addNamespace(ns uint64, pred string) string { - return x.NamespaceAttr(ns, pred) -} - -func valueToPosting_ValType(v any) (pb.Posting_ValType, error) { - switch v.(type) { - case string: - return pb.Posting_STRING, nil - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: - return pb.Posting_INT, nil - case bool: - return pb.Posting_BOOL, nil - case float32, float64: - return pb.Posting_FLOAT, nil - case []byte: - return pb.Posting_BINARY, nil - case time.Time: - return pb.Posting_DATETIME, nil - case geom.Point: - return pb.Posting_GEO, nil - case []float32, []float64: - return pb.Posting_VFLOAT, nil - default: - return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) +func getDefaultNamespace(db *DB, ns ...uint64) (*Namespace, error) { + dbOpts := &modusDbOptions{ + namespace: db.defaultNamespace.ID(), } -} - -func valueToValType(v any) (*api.Value, error) { - switch val := v.(type) { - case string: - return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil - case int: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int8: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int16: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int32: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int64: - return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil - case uint8: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint16: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint32: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case bool: - return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil - case float32: - return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil - case float64: - return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil - case []byte: - return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil - case time.Time: - bytes, err := val.MarshalBinary() - if err != nil { - return nil, err - } - return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil - case geom.Point: - bytes, err := wkb.Marshal(&val, binary.LittleEndian) - if err != nil { - return nil, err - } - return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil - case uint, uint64: - return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil - default: - return nil, fmt.Errorf("unsupported type %T", v) + for _, ns := range ns { + WithNamespace(ns)(dbOpts) } -} -func generateDqlMutationAndSchema[T any](n *Namespace, object *T, - uid uint64) ([]*dql.Mutation, *schema.ParsedSchema, error) { - t := reflect.TypeOf(*object) - if t.Kind() != reflect.Struct { - return nil, nil, fmt.Errorf("expected struct, got %s", t.Kind()) - } + return db.getNamespaceWithLock(dbOpts.namespace) +} - jsonFields, _, err := getFieldTags(t) - if err != nil { - return nil, nil, err +func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { + if len(ns) > 1 { + return 0, object, fmt.Errorf("only one namespace is allowed") } - values := getFieldValues(object, jsonFields) - sch := &schema.ParsedSchema{} + ctx := context.Background() - nquads := make([]*api.NQuad, 0) - for jsonName, value := range values { - if jsonName == "uid" { - continue - } - valType, err := valueToPosting_ValType(value) - if err != nil { - return nil, nil, err - } - sch.Preds = append(sch.Preds, &pb.SchemaUpdate{ - Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), - ValueType: valType, - }) - val, err := valueToValType(value) - if err != nil { - return nil, nil, err - } - nquad := &api.NQuad{ - Namespace: n.ID(), - Subject: fmt.Sprint(uid), - Predicate: getPredicateName(t.Name(), jsonName), - ObjectValue: val, - } - nquads = append(nquads, nquad) - } - sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: addNamespace(n.id, t.Name()), - Fields: sch.Preds, - }) + db.mutex.Lock() + defer db.mutex.Unlock() - val, err := valueToValType(t.Name()) + n, err := getDefaultNamespace(db, ns...) if err != nil { - return nil, nil, err - } - nquad := &api.NQuad{ - Namespace: n.ID(), - Subject: fmt.Sprint(uid), - Predicate: "dgraph.type", - ObjectValue: val, + return 0, object, err } - nquads = append(nquads, nquad) - - dms := make([]*dql.Mutation, 0) - dms = append(dms, &dql.Mutation{ - Set: nquads, - }) - - return dms, sch, nil -} - -func Create[T any](ctx context.Context, n *Namespace, object *T) (uint64, *T, error) { - n.db.mutex.Lock() - defer n.db.mutex.Unlock() - uid, err := n.db.z.nextUID() + gid, err := db.z.nextUID() if err != nil { return 0, object, err } - dms, sch, err := generateDqlMutationAndSchema(n, object, uid) + dms, sch, err := generateCreateDqlMutationAndSchema(n, object, gid) if err != nil { return 0, object, err } @@ -231,15 +68,15 @@ func Create[T any](ctx context.Context, n *Namespace, object *T) (uint64, *T, er return 0, object, err } - if !n.db.isOpen { + if !db.isOpen { return 0, object, ErrClosedDB } - startTs, err := n.db.z.nextTs() + startTs, err := db.z.nextTs() if err != nil { return 0, object, err } - commitTs, err := n.db.z.nextTs() + commitTs, err := db.z.nextTs() if err != nil { return 0, object, err } @@ -268,119 +105,28 @@ func Create[T any](ctx context.Context, n *Namespace, object *T) (uint64, *T, er v := reflect.ValueOf(object).Elem() - uidField := v.FieldByName("Uid") + gidField := v.FieldByName("Gid") - if uidField.IsValid() && uidField.CanSet() && uidField.Kind() == reflect.Uint64 { - uidField.SetUint(uid) + if gidField.IsValid() && gidField.CanSet() && gidField.Kind() == reflect.Uint64 { + gidField.SetUint(gid) } - return uid, object, nil -} - -func createDynamicStruct(t reflect.Type, jsonFields map[string]string) reflect.Type { - fields := make([]reflect.StructField, 0, len(jsonFields)) - for fieldName, jsonName := range jsonFields { - field, _ := t.FieldByName(fieldName) - fields = append(fields, reflect.StructField{ - Name: field.Name, - Type: field.Type, - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), - }) - } - return reflect.StructOf(fields) + return gid, object, nil } -func mapDynamicToFinal(dynamic any, final any) { - vFinal := reflect.ValueOf(final).Elem() - vDynamic := reflect.ValueOf(dynamic).Elem() - - for i := 0; i < vDynamic.NumField(); i++ { - field := vDynamic.Type().Field(i) - value := vDynamic.Field(i) - - finalField := vFinal.FieldByName(field.Name) - if finalField.IsValid() && finalField.CanSet() { - finalField.Set(value) - } +func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { + ctx := context.Background() + n, err := getDefaultNamespace(db, ns...) + if err != nil { + return 0, nil, err } -} - -func Get[T any, R UniqueField](ctx context.Context, n *Namespace, uniqueField R) (*T, error) { if uid, ok := any(uniqueField).(uint64); ok { - return getByUid[T](ctx, n, uid) + return getByGid[T](ctx, n, uid) } if cf, ok := any(uniqueField).(ConstrainedField); ok { return getByConstrainedField[T](ctx, n, cf) } - return nil, fmt.Errorf("invalid unique field type") -} - -func getByUid[T any](ctx context.Context, n *Namespace, uid uint64) (*T, error) { - query := fmt.Sprintf(` - { - obj(func: uid(%d)) { - uid - expand(_all_) - } - } - `, uid) - - return executeGet[T](ctx, n, query) -} - -func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (*T, error) { - query := fmt.Sprintf(` - { - obj(func: eq(%s, %s)) { - uid - expand(_all_) - } - } - `, cf.Key, cf.Value) - - return executeGet[T](ctx, n, query) -} - -func executeGet[T any](ctx context.Context, n *Namespace, query string) (*T, error) { - resp, err := n.Query(ctx, query) - if err != nil { - return nil, err - } - - var obj T - - t := reflect.TypeOf(obj) - - jsonFields, _, err := getFieldTags(t) - if err != nil { - return nil, err - } - - dynamicType := createDynamicStruct(t, jsonFields) - - dynamicInstance := reflect.New(dynamicType).Interface() - - var result struct { - Obj []any `json:"obj"` - } - - result.Obj = append(result.Obj, dynamicInstance) - - // Unmarshal the JSON response into the dynamic struct - if err := json.Unmarshal(resp.Json, &result); err != nil { - return nil, err - } - - // Check if we have at least one object in the response - if len(result.Obj) == 0 { - return nil, fmt.Errorf("no object found") - } - - // Map the dynamic struct to the final type T - finalObject := reflect.New(t).Interface() - mapDynamicToFinal(result.Obj[0], finalObject) - - return finalObject.(*T), nil + return 0, nil, fmt.Errorf("invalid unique field type") } diff --git a/api_helper.go b/api_helper.go new file mode 100644 index 0000000..50a84d5 --- /dev/null +++ b/api_helper.go @@ -0,0 +1,237 @@ +package modusdb + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "reflect" + "time" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/dql" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/x" + "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/wkb" +) + +func getPredicateName(typeName, fieldName string) string { + return fmt.Sprint(typeName, ".", fieldName) +} + +func addNamespace(ns uint64, pred string) string { + return x.NamespaceAttr(ns, pred) +} + +func valueToPosting_ValType(v any) (pb.Posting_ValType, error) { + switch v.(type) { + case string: + return pb.Posting_STRING, nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + return pb.Posting_INT, nil + case bool: + return pb.Posting_BOOL, nil + case float32, float64: + return pb.Posting_FLOAT, nil + case []byte: + return pb.Posting_BINARY, nil + case time.Time: + return pb.Posting_DATETIME, nil + case geom.Point: + return pb.Posting_GEO, nil + case []float32, []float64: + return pb.Posting_VFLOAT, nil + default: + return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) + } +} + +func valueToValType(v any) (*api.Value, error) { + switch val := v.(type) { + case string: + return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil + case int: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int64: + return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil + case uint8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case bool: + return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil + case float32: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil + case float64: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil + case []byte: + return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil + case time.Time: + bytes, err := val.MarshalBinary() + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil + case geom.Point: + bytes, err := wkb.Marshal(&val, binary.LittleEndian) + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil + case uint, uint64: + return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil + default: + return nil, fmt.Errorf("unsupported type %T", v) + } +} + +func generateCreateDqlMutationAndSchema[T any](n *Namespace, object *T, + gid uint64) ([]*dql.Mutation, *schema.ParsedSchema, error) { + t := reflect.TypeOf(*object) + if t.Kind() != reflect.Struct { + return nil, nil, fmt.Errorf("expected struct, got %s", t.Kind()) + } + + jsonFields, dbFields, _, err := getFieldTags(t) + if err != nil { + return nil, nil, err + } + values := getFieldValues(object, jsonFields) + sch := &schema.ParsedSchema{} + + nquads := make([]*api.NQuad, 0) + for jsonName, value := range values { + if jsonName == "gid" { + continue + } + valType, err := valueToPosting_ValType(value) + if err != nil { + return nil, nil, err + } + u := &pb.SchemaUpdate{ + Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), + ValueType: valType, + } + if dbFields[jsonName] != nil && dbFields[jsonName].constraint == "unique" { + u.Directive = pb.SchemaUpdate_INDEX + u.Tokenizer = []string{"exact"} + } + sch.Preds = append(sch.Preds, u) + val, err := valueToValType(value) + if err != nil { + return nil, nil, err + } + nquad := &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: getPredicateName(t.Name(), jsonName), + ObjectValue: val, + } + nquads = append(nquads, nquad) + } + sch.Types = append(sch.Types, &pb.TypeUpdate{ + TypeName: addNamespace(n.id, t.Name()), + Fields: sch.Preds, + }) + + val, err := valueToValType(t.Name()) + if err != nil { + return nil, nil, err + } + nquad := &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: "dgraph.type", + ObjectValue: val, + } + nquads = append(nquads, nquad) + + dms := make([]*dql.Mutation, 0) + dms = append(dms, &dql.Mutation{ + Set: nquads, + }) + + return dms, sch, nil +} + +func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, *T, error) { + query := fmt.Sprintf(` + { + obj(func: uid(%d)) { + uid + expand(_all_) + } + } + `, gid) + + return executeGet[T](ctx, n, query, nil) +} + +func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, *T, error) { + query := fmt.Sprintf(` + { + obj(func: eq(%s, %s)) { + uid + expand(_all_) + } + } + `, cf.Key, cf.Value) + + return executeGet[T](ctx, n, query, &cf) +} + +func executeGet[T any](ctx context.Context, n *Namespace, query string, cf *ConstrainedField) (uint64, *T, error) { + var obj T + + t := reflect.TypeOf(obj) + + jsonFields, dbTags, _, err := getFieldTags(t) + if err != nil { + return 0, nil, err + } + + if cf != nil && dbTags[cf.Key].constraint == "" { + return 0, nil, fmt.Errorf("constraint not defined for field %s", cf.Key) + } + + resp, err := n.Query(ctx, query) + if err != nil { + return 0, nil, err + } + + dynamicType := createDynamicStruct(t, jsonFields) + + dynamicInstance := reflect.New(dynamicType).Interface() + + var result struct { + Obj []any `json:"obj"` + } + + result.Obj = append(result.Obj, dynamicInstance) + + // Unmarshal the JSON response into the dynamic struct + if err := json.Unmarshal(resp.Json, &result); err != nil { + return 0, nil, err + } + + // Check if we have at least one object in the response + if len(result.Obj) == 0 { + return 0, nil, fmt.Errorf("no object found") + } + + // Map the dynamic struct to the final type T + finalObject := reflect.New(t).Interface() + gid := mapDynamicToFinal(result.Obj[0], finalObject) + + return gid, finalObject.(*T), nil +} diff --git a/api_reflect.go b/api_reflect.go new file mode 100644 index 0000000..817736e --- /dev/null +++ b/api_reflect.go @@ -0,0 +1,115 @@ +package modusdb + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +type dbTag struct { + constraint string +} + +func getFieldTags(t reflect.Type) (jsonTags map[string]string, jsonToDbTags map[string]*dbTag, + reverseEdgeTags map[string]string, err error) { + + jsonTags = make(map[string]string) + jsonToDbTags = make(map[string]*dbTag) + reverseEdgeTags = make(map[string]string) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + return nil, nil, nil, fmt.Errorf("field %s has no json tag", field.Name) + } + jsonName := strings.Split(jsonTag, ",")[0] + jsonTags[field.Name] = jsonName + + reverseEdgeTag := field.Tag.Get("readFrom") + if reverseEdgeTag != "" { + typeAndField := strings.Split(reverseEdgeTag, ",") + if len(typeAndField) != 2 { + return nil, nil, nil, fmt.Errorf(`field %s has invalid readFrom tag, + expected format is type=,field=`, field.Name) + } + t := strings.Split(typeAndField[0], "=")[1] + f := strings.Split(typeAndField[1], "=")[1] + reverseEdgeTags[field.Name] = getPredicateName(t, f) + } + + dbConstraintsTag := field.Tag.Get("db") + if dbConstraintsTag != "" { + jsonToDbTags[jsonName] = &dbTag{} + dbTagsSplit := strings.Split(dbConstraintsTag, ",") + for _, dbTag := range dbTagsSplit { + split := strings.Split(dbTag, "=") + if split[0] == "constraint" { + jsonToDbTags[jsonName].constraint = split[1] + } + } + } + } + return jsonTags, jsonToDbTags, reverseEdgeTags, nil +} + +func getFieldValues(object any, jsonFields map[string]string) map[string]any { + values := make(map[string]any) + v := reflect.ValueOf(object).Elem() + for fieldName, jsonName := range jsonFields { + fieldValue := v.FieldByName(fieldName) + values[jsonName] = fieldValue.Interface() + + } + return values +} + +func createDynamicStruct(t reflect.Type, jsonFields map[string]string) reflect.Type { + fields := make([]reflect.StructField, 0, len(jsonFields)) + for fieldName, jsonName := range jsonFields { + field, _ := t.FieldByName(fieldName) + if fieldName != "Gid" { + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: field.Type, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } + } + fields = append(fields, reflect.StructField{ + Name: "Uid", + Type: reflect.TypeOf(""), + Tag: reflect.StructTag(`json:"uid"`), + }) + return reflect.StructOf(fields) +} + +func mapDynamicToFinal(dynamic any, final any) uint64 { + vFinal := reflect.ValueOf(final).Elem() + vDynamic := reflect.ValueOf(dynamic).Elem() + + gid := uint64(0) + + for i := 0; i < vDynamic.NumField(); i++ { + field := vDynamic.Type().Field(i) + value := vDynamic.Field(i) + + var finalField reflect.Value + if field.Name == "Uid" { + finalField = vFinal.FieldByName("Gid") + gidStr := value.String() + gid, _ = strconv.ParseUint(gidStr, 0, 64) + } else { + finalField = vFinal.FieldByName(field.Name) + } + if finalField.IsValid() && finalField.CanSet() { + // if field name is uid, convert it to uint64 + if field.Name == "Uid" { + finalField.SetUint(gid) + } else { + finalField.Set(value) + } + } + } + return gid +} diff --git a/api_test.go b/api_test.go index ff0cb45..24d9b6e 100644 --- a/api_test.go +++ b/api_test.go @@ -10,12 +10,14 @@ import ( ) type User struct { - Uid uint64 `json:"uid,omitempty"` - Name string `json:"name,omitempty"` - Age int `json:"age,omitempty"` + Gid uint64 `json:"gid,omitempty"` + Name string `json:"name,omitempty"` + Age int `json:"age,omitempty"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` } func TestCreateApi(t *testing.T) { + ctx := context.Background() db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) defer db.Close() @@ -23,41 +25,57 @@ func TestCreateApi(t *testing.T) { db1, err := db.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(context.Background())) + require.NoError(t, db1.DropData(ctx)) user := &User{ - Name: "B", - Age: 20, + Name: "B", + Age: 20, + ClerkId: "123", } - uid, _, err := modusdb.Create(context.Background(), db1, user) + gid, _, err := modusdb.Create(db, user, db1.ID()) require.NoError(t, err) require.Equal(t, "B", user.Name) - require.Equal(t, uint64(2), uid) - require.Equal(t, uint64(2), user.Uid) + require.Equal(t, uint64(2), gid) + require.Equal(t, uint64(2), user.Gid) query := `{ me(func: has(User.name)) { uid User.name User.age + User.clerk_id } }` - resp, err := db1.Query(context.Background(), query) + resp, err := db1.Query(ctx, query) require.NoError(t, err) - require.JSONEq(t, `{"me":[{"uid":"0x2","User.name":"B","User.age":20}]}`, string(resp.GetJson())) + require.JSONEq(t, `{"me":[{"uid":"0x2","User.name":"B","User.age":20,"User.clerk_id":"123"}]}`, + string(resp.GetJson())) // TODO schema{} should work - resp, err = db1.Query(context.Background(), `schema(pred: [User.name, User.age]) {type}`) + schemaQuery := `schema(pred: [User.name, User.age, User.clerk_id]) + { + type + index + tokenizer + }` + resp, err = db1.Query(ctx, schemaQuery) require.NoError(t, err) require.JSONEq(t, - `{"schema":[{"predicate":"User.age","type":"int"},{"predicate":"User.name","type":"string"}]}`, + `{"schema": + [ + {"predicate":"User.age","type":"int"}, + {"predicate":"User.clerk_id","type":"string","index":true,"tokenizer":["exact"]}, + {"predicate":"User.name","type":"string"} + ] + }`, string(resp.GetJson())) } func TestCreateApiWithNonStruct(t *testing.T) { + ctx := context.Background() db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) defer db.Close() @@ -65,19 +83,20 @@ func TestCreateApiWithNonStruct(t *testing.T) { db1, err := db.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(context.Background())) + require.NoError(t, db1.DropData(ctx)) user := &User{ Name: "B", Age: 20, } - _, _, err = modusdb.Create[*User](context.Background(), db1, &user) + _, _, err = modusdb.Create[*User](db, &user, db1.ID()) require.Error(t, err) require.Equal(t, "expected struct, got ptr", err.Error()) } func TestGetApi(t *testing.T) { + ctx := context.Background() db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) defer db.Close() @@ -85,19 +104,21 @@ func TestGetApi(t *testing.T) { db1, err := db.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(context.Background())) + require.NoError(t, db1.DropData(ctx)) user := &User{ Name: "B", Age: 20, } - _, _, err = modusdb.Create(context.Background(), db1, user) + gid, _, err := modusdb.Create(db, user, db1.ID()) require.NoError(t, err) - userQuery, err := modusdb.Get[User](context.Background(), db1, uint64(2)) + gid, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) require.NoError(t, err) - require.Equal(t, 20, userQuery.Age) - require.Equal(t, "B", userQuery.Name) + require.Equal(t, uint64(2), gid) + require.Equal(t, uint64(2), queriedUser.Gid) + require.Equal(t, 20, queriedUser.Age) + require.Equal(t, "B", queriedUser.Name) } diff --git a/api_types.go b/api_types.go new file mode 100644 index 0000000..5d2f90d --- /dev/null +++ b/api_types.go @@ -0,0 +1,9 @@ +package modusdb + +type UniqueField interface { + uint64 | ConstrainedField +} +type ConstrainedField struct { + Key string + Value any +} diff --git a/db.go b/db.go index 6b212bf..0f32a56 100644 --- a/db.go +++ b/db.go @@ -38,7 +38,7 @@ type DB struct { z *zero // points to default / 0 / galaxy namespace - gxy *Namespace + defaultNamespace *Namespace } // New returns a new modusDB instance. @@ -77,7 +77,7 @@ func New(conf Config) (*DB, error) { x.UpdateHealthStatus(true) - db.gxy = &Namespace{id: 0, db: db} + db.defaultNamespace = &Namespace{id: 0, db: db} return db, nil } @@ -112,6 +112,10 @@ func (db *DB) GetNamespace(nsID uint64) (*Namespace, error) { db.mutex.RLock() defer db.mutex.RUnlock() + return db.getNamespaceWithLock(nsID) +} + +func (db *DB) getNamespaceWithLock(nsID uint64) (*Namespace, error) { if !db.isOpen { return nil, ErrClosedDB } @@ -150,27 +154,27 @@ func (db *DB) DropAll(ctx context.Context) error { } func (db *DB) DropData(ctx context.Context) error { - return db.gxy.DropData(ctx) + return db.defaultNamespace.DropData(ctx) } func (db *DB) AlterSchema(ctx context.Context, sch string) error { - return db.gxy.AlterSchema(ctx, sch) + return db.defaultNamespace.AlterSchema(ctx, sch) } func (db *DB) Query(ctx context.Context, q string) (*api.Response, error) { - return db.gxy.Query(ctx, q) + return db.defaultNamespace.Query(ctx, q) } func (db *DB) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { - return db.gxy.Mutate(ctx, ms) + return db.defaultNamespace.Mutate(ctx, ms) } func (db *DB) Load(ctx context.Context, schemaPath, dataPath string) error { - return db.gxy.Load(ctx, schemaPath, dataPath) + return db.defaultNamespace.Load(ctx, schemaPath, dataPath) } func (db *DB) LoadData(inCtx context.Context, dataDir string) error { - return db.gxy.LoadData(inCtx, dataDir) + return db.defaultNamespace.LoadData(inCtx, dataDir) } // Close closes the modusDB instance. From a8aee3e6f6340277595caeb5d7d9fa565a32ccf4 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 19 Dec 2024 08:36:50 -0800 Subject: [PATCH 06/23] Jai/hyp 2771 benchmark modusdb (#32) --- .gitignore | 2 + live_benchmark_test.go | 112 +++++++++++++++++++++++++++++++++++++++++ live_test.go | 16 +++--- 3 files changed, 121 insertions(+), 9 deletions(-) create mode 100644 live_benchmark_test.go diff --git a/.gitignore b/.gitignore index b3d8b55..a63304e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ go.work.sum # env file .env + +cpu_profile.prof diff --git a/live_benchmark_test.go b/live_benchmark_test.go new file mode 100644 index 0000000..5efd4b2 --- /dev/null +++ b/live_benchmark_test.go @@ -0,0 +1,112 @@ +package modusdb_test + +import ( + "context" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "testing" + + "github.com/hypermodeinc/modusdb" + "github.com/stretchr/testify/require" +) + +func BenchmarkDatabaseOperations(b *testing.B) { + setupProfiler := func(b *testing.B) *os.File { + f, err := os.Create("cpu_profile.prof") + if err != nil { + b.Fatal("could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + b.Fatal("could not start CPU profiling: ", err) + } + return f + } + + reportMemStats := func(b *testing.B, initialAlloc uint64) { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + b.ReportMetric(float64(ms.Alloc-initialAlloc)/float64(b.N), "bytes/op") + b.ReportMetric(float64(ms.NumGC), "total-gc-cycles") + } + + b.Run("DropAndLoad", func(b *testing.B) { + f := setupProfiler(b) + defer f.Close() + defer pprof.StopCPUProfile() + + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + initialAlloc := ms.Alloc + + db, err := modusdb.New(modusdb.NewDefaultConfig(b.TempDir())) + require.NoError(b, err) + defer db.Close() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + dataFolder := b.TempDir() + schemaFile := filepath.Join(dataFolder, "data.schema") + dataFile := filepath.Join(dataFolder, "data.rdf") + require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) + require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) + require.NoError(b, db.Load(context.Background(), schemaFile, dataFile)) + } + reportMemStats(b, initialAlloc) + }) + + b.Run("Query", func(b *testing.B) { + f := setupProfiler(b) + defer f.Close() + defer pprof.StopCPUProfile() + + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + initialAlloc := ms.Alloc + + // Setup database with data once + db, err := modusdb.New(modusdb.NewDefaultConfig(b.TempDir())) + require.NoError(b, err) + defer db.Close() + + dataFolder := b.TempDir() + schemaFile := filepath.Join(dataFolder, "data.schema") + dataFile := filepath.Join(dataFolder, "data.rdf") + require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) + require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) + require.NoError(b, db.Load(context.Background(), schemaFile, dataFile)) + + const query = `{ + caro(func: allofterms(name@en, "Marc Caro")) { + name@en + director.film { + name@en + } + } + }` + const expected = `{ + "caro": [ + { + "name@en": "Marc Caro", + "director.film": [ + { + "name@en": "Delicatessen" + }, + { + "name@en": "The City of Lost Children" + } + ] + } + ] + }` + + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := db.Query(context.Background(), query) + require.NoError(b, err) + require.JSONEq(b, expected, string(resp.Json)) + } + reportMemStats(b, initialAlloc) + }) +} diff --git a/live_test.go b/live_test.go index b1e173b..d6b4880 100644 --- a/live_test.go +++ b/live_test.go @@ -19,15 +19,11 @@ const ( baseURL = "https://github.com/dgraph-io/benchmarks/blob/master/data" oneMillionSchema = baseURL + "/1million.schema?raw=true" oneMillionRDF = baseURL + "/1million.rdf.gz?raw=true" -) - -func TestLiveLoaderSmall(t *testing.T) { - const ( - dbSchema = ` + DbSchema = ` director.film : [uid] @reverse @count . name : string @index(hash, term, trigram, fulltext) @lang . ` - data = ` + SmallData = ` <12534504120601169429> "Marc Caro"@en . <2698880893682087932> "Delicatessen"@en . <2698880893682087932> "Delicatessen"@de . @@ -40,7 +36,9 @@ func TestLiveLoaderSmall(t *testing.T) { <12534504120601169429> <15617393957106514527> . <14514306440537019930> <15617393957106514527> . ` - ) +) + +func TestLiveLoaderSmall(t *testing.T) { db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) @@ -49,8 +47,8 @@ func TestLiveLoaderSmall(t *testing.T) { dataFolder := t.TempDir() schemaFile := filepath.Join(dataFolder, "data.schema") dataFile := filepath.Join(dataFolder, "data.rdf") - require.NoError(t, os.WriteFile(schemaFile, []byte(dbSchema), 0600)) - require.NoError(t, os.WriteFile(dataFile, []byte(data), 0600)) + require.NoError(t, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) + require.NoError(t, os.WriteFile(dataFile, []byte(SmallData), 0600)) require.NoError(t, db.Load(context.Background(), schemaFile, dataFile)) const query = `{ From 2e3b10633a4c5d46b9d4041ec1bdd683e2262963 Mon Sep 17 00:00:00 2001 From: Aman Mangal Date: Fri, 20 Dec 2024 10:34:20 +0530 Subject: [PATCH 07/23] upgrade dependencies in go.mod (#24) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 63 +++++++++++----------- go.sum | 160 ++++++++++++++++++++++++++------------------------------ zero.go | 25 ++++----- 3 files changed, 120 insertions(+), 128 deletions(-) diff --git a/go.mod b/go.mod index 8966a93..877c2e5 100644 --- a/go.mod +++ b/go.mod @@ -6,25 +6,25 @@ toolchain go1.23.3 require ( github.com/cavaliergopher/grab/v3 v3.0.1 - github.com/dgraph-io/badger/v4 v4.4.0 - github.com/dgraph-io/dgo/v240 v240.0.1 - github.com/dgraph-io/dgraph/v24 v24.0.3-0.20241202011806-64256ce6cac9 - github.com/dgraph-io/ristretto/v2 v2.0.0 + github.com/dgraph-io/badger/v4 v4.5.0 + github.com/dgraph-io/dgo/v240 v240.1.0 + github.com/dgraph-io/dgraph/v24 v24.0.3-0.20241213192353-fd411c5b915b + github.com/dgraph-io/ristretto/v2 v2.0.1 github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.10.0 github.com/twpayne/go-geom v1.5.7 golang.org/x/sync v0.10.0 + google.golang.org/protobuf v1.35.2 ) require ( contrib.go.opencensus.io/exporter/jaeger v0.2.1 // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect - github.com/DataDog/datadog-go v4.8.3+incompatible // indirect + github.com/DataDog/datadog-go v3.5.0+incompatible // indirect github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20220622145613-731d59e8b567 // indirect github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect github.com/IBM/sarama v1.43.3 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/agnivade/levenshtein v1.1.1 // indirect + github.com/agnivade/levenshtein v1.0.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.17.0 // indirect github.com/blevesearch/bleve/v2 v2.4.3 // indirect @@ -36,23 +36,23 @@ require ( github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/chewxy/math32 v1.11.0 // indirect + github.com/chewxy/math32 v1.10.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/gqlgen v0.13.2 // indirect github.com/dgraph-io/gqlparser/v2 v2.2.2 // indirect github.com/dgraph-io/simdjson-go v0.3.0 // indirect - github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da // indirect github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/felixge/fgprof v0.9.5 // indirect + github.com/felixge/fgprof v0.9.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/getsentry/sentry-go v0.29.1 // indirect + github.com/getsentry/sentry-go v0.30.0 // indirect github.com/go-jose/go-jose/v4 v4.0.4 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect @@ -62,17 +62,17 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/codesearch v1.2.0 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect - github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect + github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect - github.com/hashicorp/go-sockaddr v1.0.6 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/vault/api v1.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -82,11 +82,12 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid v1.2.3 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/md5-simd v1.1.0 // indirect github.com/minio/minio-go/v6 v6.0.57 // indirect - github.com/minio/sha256-simd v1.0.1 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/panicwrap v1.0.0 // indirect @@ -94,27 +95,29 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect + github.com/philhofer/fwd v1.1.2 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/profile v1.7.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.20.5 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.59.1 // indirect + github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/prometheus/statsd_exporter v0.27.1 // indirect + github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/soheilhy/cmux v0.1.5 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/cobra v1.8.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tinylib/msgp v1.2.1 // indirect - github.com/uber/jaeger-client-go v2.28.0+incompatible // indirect + github.com/tinylib/msgp v1.1.8 // indirect + github.com/uber/jaeger-client-go v2.25.0+incompatible // indirect github.com/viterin/partial v1.1.0 // indirect github.com/viterin/vek v0.4.2 // indirect github.com/xdg/scram v1.0.5 // indirect @@ -123,18 +126,18 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect - golang.org/x/net v0.31.0 // indirect + golang.org/x/crypto v0.30.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/net v0.32.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect + gonum.org/v1/gonum v0.12.0 // indirect google.golang.org/api v0.196.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.68.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect - gopkg.in/DataDog/dd-trace-go.v1 v1.67.1 // indirect + google.golang.org/grpc v1.68.1 // indirect + gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 1f71c4e..d91f346 100644 --- a/go.sum +++ b/go.sum @@ -41,22 +41,20 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= +github.com/DataDog/datadog-go v3.5.0+incompatible h1:AShr9cqkF+taHjyQgcBcQUt/ZNK+iPq4ROaZwSX5c/U= github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q= -github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20220622145613-731d59e8b567 h1:Z7zdcyzme2egv0lC43X1Q/+DxHjZflQCnJXX0mDp7+I= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20220622145613-731d59e8b567/go.mod h1:/VV3EFO/hTNQZHAqaj+CPGy2+ioFrP4EX3iRwozubhQ= github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/agnivade/levenshtein v1.0.3 h1:M5ZnqLOoZR8ygVq0FfkXsNOKzMCk0xRiow0R5+5VkQ0= github.com/agnivade/levenshtein v1.0.3/go.mod h1:4SFRZbbXWLF4MU1T9Qg0pGgH3Pjs+t6ie5efyrwRJXs= -github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= -github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= @@ -107,17 +105,11 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chewxy/math32 v1.11.0 h1:8sek2JWqeaKkVnHa7bPVqCEOUPbARo4SGxs6toKyAOo= -github.com/chewxy/math32 v1.11.0/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= -github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= -github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU= +github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= @@ -134,34 +126,33 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v4 v4.4.0 h1:rA48XiDynZLyMdlaJl67p9+lqfqwxlgKtCpYLAio7Zk= -github.com/dgraph-io/badger/v4 v4.4.0/go.mod h1:sONMmPPfbnj9FPwS/etCqky/ULth6CQJuAZSuWCmixE= -github.com/dgraph-io/dgo/v240 v240.0.1 h1:R0d9Cao3MOghrC9RVXshw6v8Jr/IjKgU2mK9sR9nclc= -github.com/dgraph-io/dgo/v240 v240.0.1/go.mod h1:urpjhWGdYVSVQAwd000iu4wHyHPpuHpwJ7aILsuGF5A= -github.com/dgraph-io/dgraph/v24 v24.0.3-0.20241202011806-64256ce6cac9 h1:6ink3iffWaAqHCOX8j35oC8+K2oiDFLwsyNSd40YmBQ= -github.com/dgraph-io/dgraph/v24 v24.0.3-0.20241202011806-64256ce6cac9/go.mod h1:2e/yPl+J7eEl9eaeeYSGMwuiQAxVh9x2Gm1SsaVvM3o= +github.com/dgraph-io/badger/v4 v4.5.0 h1:TeJE3I1pIWLBjYhIYCA1+uxrjWEoJXImFBMEBVSm16g= +github.com/dgraph-io/badger/v4 v4.5.0/go.mod h1:ysgYmIeG8dS/E8kwxT7xHyc7MkmwNYLRoYnFbr7387A= +github.com/dgraph-io/dgo/v240 v240.1.0 h1:xd8z9kEXDWOAblaLJ2HLg2tXD6ngMQwq3ehLUS7GKNg= +github.com/dgraph-io/dgo/v240 v240.1.0/go.mod h1:r8WASETKfodzKqThSAhhTNIzcEMychArKKlZXQufWuA= +github.com/dgraph-io/dgraph/v24 v24.0.3-0.20241213192353-fd411c5b915b h1:y5i9HO4AS+i1RKRGXrHm1x62zsLKDkGKB417nUgvzCk= +github.com/dgraph-io/dgraph/v24 v24.0.3-0.20241213192353-fd411c5b915b/go.mod h1:AFALH9qEz1P9ZmOeB6/jhvcKhomttcXRL8r6XptHFYg= github.com/dgraph-io/gqlgen v0.13.2 h1:TNhndk+eHKj5qE7BenKKSYdSIdOGhLqxR1rCiMso9KM= github.com/dgraph-io/gqlgen v0.13.2/go.mod h1:iCOrOv9lngN7KAo+jMgvUPVDlYHdf7qDwsTkQby2Sis= github.com/dgraph-io/gqlparser/v2 v2.1.1/go.mod h1:MYS4jppjyx8b9tuUtjV7jU1UFZK6P9fvO8TsIsQtRKU= github.com/dgraph-io/gqlparser/v2 v2.2.2 h1:CnxXOKL4EPguKqcGV/z4u4VoW5izUkOTIsNM6xF+0f4= github.com/dgraph-io/gqlparser/v2 v2.2.2/go.mod h1:MYS4jppjyx8b9tuUtjV7jU1UFZK6P9fvO8TsIsQtRKU= -github.com/dgraph-io/ristretto/v2 v2.0.0 h1:l0yiSOtlJvc0otkqyMaDNysg8E9/F/TYZwMbxscNOAQ= -github.com/dgraph-io/ristretto/v2 v2.0.0/go.mod h1:FVFokF2dRqXyPyeMnK1YDy8Fc6aTe0IKgbcd03CYeEk= +github.com/dgraph-io/ristretto/v2 v2.0.1 h1:7W0LfEP+USCmtrUjJsk+Jv2jbhJmb72N4yRI7GrLdMI= +github.com/dgraph-io/ristretto/v2 v2.0.1/go.mod h1:K7caLeufSdxm+ITp1n/73U+VbFVAHrexfLbz4n14hpo= github.com/dgraph-io/simdjson-go v0.3.0 h1:h71LO7vR4LHMPUhuoGN8bqGm1VNfGOlAG8BI6iDUKw0= github.com/dgraph-io/simdjson-go v0.3.0/go.mod h1:Otpysdjaxj9OGaJusn4pgQV7OFh2bELuHANq0I78uvY= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= -github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa512G+w+Pxci9hJPB8oMnkcP3iZF38= +github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c h1:cHaw4wmusVzAZLEPWOCCGCfu6UvFXx9UboCHQCnjvxY= github.com/dgryski/go-groupvarint v0.0.0-20230630160417-2bfb7969fb3c/go.mod h1:MlkUQveSLEDbIgq2r1e++tSf0zfzU9mQpa9Qkczl+9Y= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dgryski/trifles v0.0.0-20190318185328-a8d75aae118c h1:TUuUh0Xgj97tLMNtWtNvI9mIV6isjEb9lBMNv+77IGM= github.com/dgryski/trifles v0.0.0-20190318185328-a8d75aae118c/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= -github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v27.4.0+incompatible h1:I9z7sQ5qyzO0BfAb9IMOawRkAGxhYsidKiTMcm0DU+A= +github.com/docker/docker v27.4.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -182,9 +173,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= -github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= -github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -193,8 +183,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA= -github.com/getsentry/sentry-go v0.29.1/go.mod h1:x3AtIzN01d6SiWkderzaH28Tm0lgkafpJ5Bm3li39O0= +github.com/getsentry/sentry-go v0.30.0 h1:lWUwDnY7sKHaVIoZ9wYqRHJ5iEmoc0pqcRqFkosKzBo= +github.com/getsentry/sentry-go v0.30.0/go.mod h1:WU9B9/1/sHDqeV8T+3VwwbjeR5MSXs/6aqG3mqZrezA= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -213,9 +203,8 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= -github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -223,9 +212,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -303,9 +289,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 h1:c5FlPPgxOn7kJz3VoPLkQYQXGBS3EklQ4Zfi57uOuqQ= -github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -342,13 +327,14 @@ github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFO github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 h1:iBt4Ew4XEGLfh6/bPk4rSYmuZJGizr6/x/AEizP0CQc= -github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8/go.mod h1:aiJI+PIApBRQG7FZTEBx5GiiX+HbOHilUdNxUZi4eV0= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 h1:om4Al8Oy7kCm/B86rLCLah4Dt5Aa0Fr5rYBG60OzwHQ= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.6 h1:RSG8rKU28VTUTvEKghe5gIhIQpv8evvNpnDEyqO4u9I= -github.com/hashicorp/go-sockaddr v1.0.6/go.mod h1:uoUUmtwU7n9Dv3O4SNLeFvg0SxQ3lyjsj6+CCykpaxI= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -358,9 +344,8 @@ github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= -github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= @@ -371,7 +356,6 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= @@ -387,7 +371,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -408,9 +391,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -425,12 +409,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -443,24 +425,25 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= -github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/panicwrap v1.0.0 h1:67zIyVakCIvcs69A0FGfZjBdPleaonSgGlXRSRlb6fE= @@ -488,14 +471,13 @@ github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKw github.com/opentracing/basictracer-go v1.1.0/go.mod h1:V2HZueSJEp879yv285Aap1BS69fQMD+MNP1mRs6mBQc= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= -github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 h1:jYi87L8j62qkXzaYHAQAhEapgukhenIMZRBKTNRLHJ4= -github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= @@ -534,8 +516,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= -github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -545,9 +527,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= -github.com/prometheus/statsd_exporter v0.27.1 h1:tcRJOmwlA83HPfWzosAgr2+zEN5XDFv+M2mn/uYkn5Y= -github.com/prometheus/statsd_exporter v0.27.1/go.mod h1:vA6ryDfsN7py/3JApEst6nLTJboq66XsNcJGNmC88NQ= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -559,11 +540,13 @@ github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= @@ -586,9 +569,8 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -617,14 +599,13 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tinylib/msgp v1.2.1 h1:6ypy2qcCznxpP4hpORzhtXyTqrBs7cfM9MCCWY8zsmU= -github.com/tinylib/msgp v1.2.1/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twpayne/go-geom v1.5.7 h1:7fdceDUr03/MP7rAKOaTV6x9njMiQdxB/D0PDzMTCDc= github.com/twpayne/go-geom v1.5.7/go.mod h1:y4fTAQtLedXW8eG2Yo4tYrIGN1yIwwKkmA+K3iSHKBA= +github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= -github.com/uber/jaeger-client-go v2.28.0+incompatible h1:G4QSBfvPKvg5ZM2j9MrJFdfI5iSljY/WnJqOGFao6HI= -github.com/uber/jaeger-client-go v2.28.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= github.com/vektah/gqlparser/v2 v2.1.0/go.mod h1:SyUiHgLATUR8BiYURfTirrTcGpcE+4XkV2se04Px1Ms= @@ -682,8 +663,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -697,8 +678,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= -golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -721,6 +702,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -764,10 +746,11 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -787,6 +770,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -834,16 +818,16 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= @@ -854,6 +838,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= @@ -914,13 +899,17 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -991,8 +980,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1007,11 +996,10 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +gopkg.in/DataDog/dd-trace-go.v1 v1.22.0 h1:gpWsqqkwUldNZXGJqT69NU9MdEDhLboK1C4nMgR0MWw= gopkg.in/DataDog/dd-trace-go.v1 v1.22.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= -gopkg.in/DataDog/dd-trace-go.v1 v1.67.1 h1:frgcpZ18wmpj+/TwyDJM8057M65aOdgaxLiZ8pb1PFU= -gopkg.in/DataDog/dd-trace-go.v1 v1.67.1/go.mod h1:6DdiJPKOeJfZyd/IUGCAd5elY8qPGkztK6wbYYsMjag= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/zero.go b/zero.go index 786724a..d790a70 100644 --- a/zero.go +++ b/zero.go @@ -8,6 +8,7 @@ import ( "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/worker" "github.com/dgraph-io/dgraph/v24/x" + "google.golang.org/protobuf/proto" ) const ( @@ -24,7 +25,7 @@ const ( zeroStateKey = "0-dgraph.modusdb.zero" ) -func (db *DB) LeaseUIDs(numUIDs uint64) (pb.AssignedIds, error) { +func (db *DB) LeaseUIDs(numUIDs uint64) (*pb.AssignedIds, error) { num := &pb.Num{Val: numUIDs, Type: pb.Num_UID} return db.z.nextUIDs(num) } @@ -98,24 +99,24 @@ func (z *zero) nextUID() (uint64, error) { return uids.StartId, nil } -func (z *zero) nextUIDs(num *pb.Num) (pb.AssignedIds, error) { - var resp pb.AssignedIds +func (z *zero) nextUIDs(num *pb.Num) (*pb.AssignedIds, error) { + var resp *pb.AssignedIds if num.Bump { if z.minLeasedUID >= num.Val { - resp = pb.AssignedIds{StartId: z.minLeasedUID, EndId: z.minLeasedUID} + resp = &pb.AssignedIds{StartId: z.minLeasedUID, EndId: z.minLeasedUID} z.minLeasedUID += 1 } else { - resp = pb.AssignedIds{StartId: z.minLeasedUID, EndId: num.Val} + resp = &pb.AssignedIds{StartId: z.minLeasedUID, EndId: num.Val} z.minLeasedUID = num.Val + 1 } } else { - resp = pb.AssignedIds{StartId: z.minLeasedUID, EndId: z.minLeasedUID + num.Val - 1} + resp = &pb.AssignedIds{StartId: z.minLeasedUID, EndId: z.minLeasedUID + num.Val - 1} z.minLeasedUID += num.Val } for z.minLeasedUID >= z.maxLeasedUID { if err := z.leaseUIDs(); err != nil { - return pb.AssignedIds{}, err + return nil, err } } @@ -143,20 +144,20 @@ func readZeroState() (*pb.MembershipState, error) { return nil, fmt.Errorf("error getting zero state: %v", err) } - var zeroState pb.MembershipState + zeroState := &pb.MembershipState{} err = item.Value(func(val []byte) error { - return zeroState.Unmarshal(val) + return proto.Unmarshal(val, zeroState) }) if err != nil { return nil, fmt.Errorf("error unmarshalling zero state: %v", err) } - return &zeroState, nil + return zeroState, nil } func (z *zero) writeZeroState() error { - zeroState := pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastNS} - data, err := zeroState.Marshal() + zeroState := &pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastNS} + data, err := proto.Marshal(zeroState) if err != nil { return fmt.Errorf("error marshalling zero state: %w", err) } From 3e824e2729b6c6c81ec6c761b18250ea04aeb123 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Fri, 20 Dec 2024 09:05:49 -0800 Subject: [PATCH 08/23] add delete api (#33) --- api.go | 98 +++++++++++++++++++++++++------------------------- api_helper.go | 74 +++++++++++++++++++++++++++++++++++--- api_reflect.go | 13 +++++-- api_test.go | 76 +++++++++++++++++++++++++++++++++++++-- api_types.go | 6 ++++ namespace.go | 4 +++ 6 files changed, 213 insertions(+), 58 deletions(-) diff --git a/api.go b/api.go index db07298..88a42b2 100644 --- a/api.go +++ b/api.go @@ -5,9 +5,6 @@ import ( "fmt" "reflect" - "github.com/dgraph-io/dgraph/v24/protos/pb" - "github.com/dgraph-io/dgraph/v24/query" - "github.com/dgraph-io/dgraph/v24/worker" "github.com/dgraph-io/dgraph/v24/x" ) @@ -23,7 +20,7 @@ func WithNamespace(namespace uint64) ModusDbOption { } } -func getDefaultNamespace(db *DB, ns ...uint64) (*Namespace, error) { +func getDefaultNamespace(db *DB, ns ...uint64) (context.Context, *Namespace, error) { dbOpts := &modusDbOptions{ namespace: db.defaultNamespace.ID(), } @@ -31,36 +28,38 @@ func getDefaultNamespace(db *DB, ns ...uint64) (*Namespace, error) { WithNamespace(ns)(dbOpts) } - return db.getNamespaceWithLock(dbOpts.namespace) + n, err := db.getNamespaceWithLock(dbOpts.namespace) + if err != nil { + return nil, nil, err + } + + ctx := context.Background() + ctx = x.AttachNamespace(ctx, n.ID()) + + return ctx, n, nil } func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { + db.mutex.Lock() + defer db.mutex.Unlock() if len(ns) > 1 { return 0, object, fmt.Errorf("only one namespace is allowed") } - ctx := context.Background() - - db.mutex.Lock() - defer db.mutex.Unlock() - - n, err := getDefaultNamespace(db, ns...) + ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { return 0, object, err } + gid, err := db.z.nextUID() if err != nil { return 0, object, err } - dms, sch, err := generateCreateDqlMutationAndSchema(n, object, gid) + dms, sch, err := generateCreateDqlMutationsAndSchema(n, object, gid) if err != nil { return 0, object, err } - edges, err := query.ToDirectedEdges(dms, nil) - if err != nil { - return 0, object, err - } ctx = x.AttachNamespace(ctx, n.ID()) err = n.alterSchemaWithParsed(ctx, sch) @@ -68,37 +67,7 @@ func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { return 0, object, err } - if !db.isOpen { - return 0, object, ErrClosedDB - } - - startTs, err := db.z.nextTs() - if err != nil { - return 0, object, err - } - commitTs, err := db.z.nextTs() - if err != nil { - return 0, object, err - } - - m := &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Edges: edges, - } - m.Edges, err = query.ExpandEdges(ctx, m) - if err != nil { - return 0, object, fmt.Errorf("error expanding edges: %w", err) - } - - p := &pb.Proposal{Mutations: m, StartTs: startTs} - if err := worker.ApplyMutations(ctx, p); err != nil { - return 0, object, err - } - - err = worker.ApplyCommited(ctx, &pb.OracleDelta{ - Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, - }) + err = applyDqlMutations(ctx, db, dms) if err != nil { return 0, object, err } @@ -115,8 +84,9 @@ func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { } func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { - ctx := context.Background() - n, err := getDefaultNamespace(db, ns...) + db.mutex.Lock() + defer db.mutex.Unlock() + ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { return 0, nil, err } @@ -130,3 +100,33 @@ func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, return 0, nil, fmt.Errorf("invalid unique field type") } + +func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { + db.mutex.Lock() + defer db.mutex.Unlock() + ctx, n, err := getDefaultNamespace(db, ns...) + if err != nil { + return 0, nil, err + } + if uid, ok := any(uniqueField).(uint64); ok { + uid, obj, err := getByGid[T](ctx, n, uid) + if err != nil { + return 0, nil, err + } + + dms := generateDeleteDqlMutations(n, uid) + + err = applyDqlMutations(ctx, db, dms) + if err != nil { + return 0, nil, err + } + + return uid, obj, nil + } + + if cf, ok := any(uniqueField).(ConstrainedField); ok { + return getByConstrainedField[T](ctx, n, cf) + } + + return 0, nil, fmt.Errorf("invalid unique field type") +} diff --git a/api_helper.go b/api_helper.go index 50a84d5..0cda75b 100644 --- a/api_helper.go +++ b/api_helper.go @@ -11,7 +11,9 @@ import ( "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/query" "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/worker" "github.com/dgraph-io/dgraph/v24/x" "github.com/twpayne/go-geom" "github.com/twpayne/go-geom/encoding/wkb" @@ -95,7 +97,7 @@ func valueToValType(v any) (*api.Value, error) { } } -func generateCreateDqlMutationAndSchema[T any](n *Namespace, object *T, +func generateCreateDqlMutationsAndSchema[T any](n *Namespace, object *T, gid uint64) ([]*dql.Mutation, *schema.ParsedSchema, error) { t := reflect.TypeOf(*object) if t.Kind() != reflect.Struct { @@ -164,12 +166,28 @@ func generateCreateDqlMutationAndSchema[T any](n *Namespace, object *T, return dms, sch, nil } +func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { + return []*dql.Mutation{{ + Del: []*api.NQuad{ + { + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: x.Star, + ObjectValue: &api.Value{ + Val: &api.Value_DefaultVal{DefaultVal: x.Star}, + }, + }, + }, + }} +} + func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, *T, error) { query := fmt.Sprintf(` { obj(func: uid(%d)) { uid expand(_all_) + dgraph.type } } `, gid) @@ -178,14 +196,18 @@ func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, *T, } func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, *T, error) { + var obj T + + t := reflect.TypeOf(obj) query := fmt.Sprintf(` { obj(func: eq(%s, %s)) { uid expand(_all_) + dgraph.type } } - `, cf.Key, cf.Value) + `, getPredicateName(t.Name(), cf.Key), cf.Value) return executeGet[T](ctx, n, query, &cf) } @@ -204,7 +226,7 @@ func executeGet[T any](ctx context.Context, n *Namespace, query string, cf *Cons return 0, nil, fmt.Errorf("constraint not defined for field %s", cf.Key) } - resp, err := n.Query(ctx, query) + resp, err := n.queryWithLock(ctx, query) if err != nil { return 0, nil, err } @@ -226,12 +248,54 @@ func executeGet[T any](ctx context.Context, n *Namespace, query string, cf *Cons // Check if we have at least one object in the response if len(result.Obj) == 0 { - return 0, nil, fmt.Errorf("no object found") + return 0, nil, ErrNoObjFound } // Map the dynamic struct to the final type T finalObject := reflect.New(t).Interface() - gid := mapDynamicToFinal(result.Obj[0], finalObject) + gid, err := mapDynamicToFinal(result.Obj[0], finalObject) + if err != nil { + return 0, nil, err + } return gid, finalObject.(*T), nil } + +func applyDqlMutations(ctx context.Context, db *DB, dms []*dql.Mutation) error { + edges, err := query.ToDirectedEdges(dms, nil) + if err != nil { + return err + } + + if !db.isOpen { + return ErrClosedDB + } + + startTs, err := db.z.nextTs() + if err != nil { + return err + } + commitTs, err := db.z.nextTs() + if err != nil { + return err + } + + m := &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Edges: edges, + } + m.Edges, err = query.ExpandEdges(ctx, m) + if err != nil { + return fmt.Errorf("error expanding edges: %w", err) + } + + p := &pb.Proposal{Mutations: m, StartTs: startTs} + if err := worker.ApplyMutations(ctx, p); err != nil { + return err + } + + return worker.ApplyCommited(ctx, &pb.OracleDelta{ + Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, + }) +} diff --git a/api_reflect.go b/api_reflect.go index 817736e..4c84813 100644 --- a/api_reflect.go +++ b/api_reflect.go @@ -80,11 +80,15 @@ func createDynamicStruct(t reflect.Type, jsonFields map[string]string) reflect.T Name: "Uid", Type: reflect.TypeOf(""), Tag: reflect.StructTag(`json:"uid"`), + }, reflect.StructField{ + Name: "DgraphType", + Type: reflect.TypeOf([]string{}), + Tag: reflect.StructTag(`json:"dgraph.type"`), }) return reflect.StructOf(fields) } -func mapDynamicToFinal(dynamic any, final any) uint64 { +func mapDynamicToFinal(dynamic any, final any) (uint64, error) { vFinal := reflect.ValueOf(final).Elem() vDynamic := reflect.ValueOf(dynamic).Elem() @@ -99,6 +103,11 @@ func mapDynamicToFinal(dynamic any, final any) uint64 { finalField = vFinal.FieldByName("Gid") gidStr := value.String() gid, _ = strconv.ParseUint(gidStr, 0, 64) + } else if field.Name == "DgraphType" { + fieldArr := value.Interface().([]string) + if len(fieldArr) == 0 { + return 0, ErrNoObjFound + } } else { finalField = vFinal.FieldByName(field.Name) } @@ -111,5 +120,5 @@ func mapDynamicToFinal(dynamic any, final any) uint64 { } } } - return gid + return gid, nil } diff --git a/api_test.go b/api_test.go index 24d9b6e..5b1c6bc 100644 --- a/api_test.go +++ b/api_test.go @@ -107,8 +107,9 @@ func TestGetApi(t *testing.T) { require.NoError(t, db1.DropData(ctx)) user := &User{ - Name: "B", - Age: 20, + Name: "B", + Age: 20, + ClerkId: "123", } gid, _, err := modusdb.Create(db, user, db1.ID()) @@ -121,4 +122,75 @@ func TestGetApi(t *testing.T) { require.Equal(t, uint64(2), queriedUser.Gid) require.Equal(t, 20, queriedUser.Age) require.Equal(t, "B", queriedUser.Name) + require.Equal(t, "123", queriedUser.ClerkId) +} + +func TestGetApiWithConstrainedField(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + user := &User{ + Name: "B", + Age: 20, + ClerkId: "123", + } + + _, _, err = modusdb.Create(db, user, db1.ID()) + require.NoError(t, err) + + gid, queriedUser, err := modusdb.Get[User](db, modusdb.ConstrainedField{ + Key: "clerk_id", + Value: "123", + }, db1.ID()) + + require.NoError(t, err) + require.Equal(t, uint64(2), gid) + require.Equal(t, uint64(2), queriedUser.Gid) + require.Equal(t, 20, queriedUser.Age) + require.Equal(t, "B", queriedUser.Name) + require.Equal(t, "123", queriedUser.ClerkId) +} + +func TestDeleteApi(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + user := &User{ + Name: "B", + Age: 20, + ClerkId: "123", + } + + gid, _, err := modusdb.Create(db, user, db1.ID()) + require.NoError(t, err) + + _, _, err = modusdb.Delete[User](db, gid, db1.ID()) + require.NoError(t, err) + + _, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) + require.Error(t, err) + require.Equal(t, "no object found", err.Error()) + require.Nil(t, queriedUser) + + _, queriedUser, err = modusdb.Get[User](db, modusdb.ConstrainedField{ + Key: "clerk_id", + Value: "123", + }, db1.ID()) + require.Error(t, err) + require.Equal(t, "no object found", err.Error()) + require.Nil(t, queriedUser) } diff --git a/api_types.go b/api_types.go index 5d2f90d..9cea571 100644 --- a/api_types.go +++ b/api_types.go @@ -1,5 +1,11 @@ package modusdb +import "fmt" + +var ( + ErrNoObjFound = fmt.Errorf("no object found") +) + type UniqueField interface { uint64 | ConstrainedField } diff --git a/namespace.go b/namespace.go index 9668804..6d9f1af 100644 --- a/namespace.go +++ b/namespace.go @@ -173,6 +173,10 @@ func (n *Namespace) Query(ctx context.Context, query string) (*api.Response, err n.db.mutex.RLock() defer n.db.mutex.RUnlock() + return n.queryWithLock(ctx, query) +} + +func (n *Namespace) queryWithLock(ctx context.Context, query string) (*api.Response, error) { if !n.db.isOpen { return nil, ErrClosedDB } From 1116c7c673f4f8aab0998ebd1ab328e59243d408 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 26 Dec 2024 09:06:04 -0800 Subject: [PATCH 09/23] add nested mutation, getting nested values, upsert, nested types support (#35) --- api.go | 135 ++++++++++++------ api_dql.go | 41 ++++++ api_helper.go | 301 --------------------------------------- api_mutate_helper.go | 240 +++++++++++++++++++++++++++++++ api_query_helper.go | 123 ++++++++++++++++ api_reflect.go | 137 ++++++++++++++---- api_test.go | 332 ++++++++++++++++++++++++++++++++++++++++++- api_types.go | 121 +++++++++++++++- utils.go | 15 ++ 9 files changed, 1062 insertions(+), 383 deletions(-) create mode 100644 api_dql.go delete mode 100644 api_helper.go create mode 100644 api_mutate_helper.go create mode 100644 api_query_helper.go create mode 100644 utils.go diff --git a/api.go b/api.go index 88a42b2..7fcdccd 100644 --- a/api.go +++ b/api.go @@ -1,91 +1,127 @@ package modusdb import ( - "context" "fmt" - "reflect" - "github.com/dgraph-io/dgraph/v24/x" + "github.com/dgraph-io/dgraph/v24/dql" + "github.com/dgraph-io/dgraph/v24/schema" ) -type ModusDbOption func(*modusDbOptions) - -type modusDbOptions struct { - namespace uint64 -} - -func WithNamespace(namespace uint64) ModusDbOption { - return func(o *modusDbOptions) { - o.namespace = namespace +func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { + db.mutex.Lock() + defer db.mutex.Unlock() + if len(ns) > 1 { + return 0, object, fmt.Errorf("only one namespace is allowed") + } + ctx, n, err := getDefaultNamespace(db, ns...) + if err != nil { + return 0, object, err } -} -func getDefaultNamespace(db *DB, ns ...uint64) (context.Context, *Namespace, error) { - dbOpts := &modusDbOptions{ - namespace: db.defaultNamespace.ID(), + gid, err := db.z.nextUID() + if err != nil { + return 0, object, err } - for _, ns := range ns { - WithNamespace(ns)(dbOpts) + + dms := make([]*dql.Mutation, 0) + sch := &schema.ParsedSchema{} + err = generateCreateDqlMutationsAndSchema[T](ctx, n, *object, gid, &dms, sch) + if err != nil { + return 0, object, err } - n, err := db.getNamespaceWithLock(dbOpts.namespace) + err = n.alterSchemaWithParsed(ctx, sch) if err != nil { - return nil, nil, err + return 0, object, err } - ctx := context.Background() - ctx = x.AttachNamespace(ctx, n.ID()) + err = applyDqlMutations(ctx, db, dms) + if err != nil { + return 0, object, err + } - return ctx, n, nil + return getByGid[T](ctx, n, gid) } -func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { +func Upsert[T any](db *DB, object *T, ns ...uint64) (uint64, *T, bool, error) { + + var wasFound bool db.mutex.Lock() defer db.mutex.Unlock() if len(ns) > 1 { - return 0, object, fmt.Errorf("only one namespace is allowed") + return 0, object, false, fmt.Errorf("only one namespace is allowed") } + if object == nil { + return 0, nil, false, fmt.Errorf("object is nil") + } + ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { - return 0, object, err + return 0, object, false, err } - gid, err := db.z.nextUID() + gid, cf, err := getUniqueConstraint[T](*object) if err != nil { - return 0, object, err + return 0, nil, false, err } - dms, sch, err := generateCreateDqlMutationsAndSchema(n, object, gid) + dms := make([]*dql.Mutation, 0) + sch := &schema.ParsedSchema{} + err = generateCreateDqlMutationsAndSchema[T](ctx, n, *object, gid, &dms, sch) if err != nil { - return 0, object, err + return 0, nil, false, err } - ctx = x.AttachNamespace(ctx, n.ID()) - err = n.alterSchemaWithParsed(ctx, sch) if err != nil { - return 0, object, err + return 0, nil, false, err } - err = applyDqlMutations(ctx, db, dms) - if err != nil { - return 0, object, err + if gid != 0 { + gid, _, err = getByGidWithObject[T](ctx, n, gid, *object) + if err != nil && err != ErrNoObjFound { + return 0, nil, false, err + } + wasFound = err == nil + } else if cf != nil { + gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, *object) + if err != nil && err != ErrNoObjFound { + return 0, nil, false, err + } + wasFound = err == nil + } + if gid == 0 { + gid, err = db.z.nextUID() + if err != nil { + return 0, nil, false, err + } } - v := reflect.ValueOf(object).Elem() + dms = make([]*dql.Mutation, 0) + err = generateCreateDqlMutationsAndSchema[T](ctx, n, *object, gid, &dms, sch) + if err != nil { + return 0, nil, false, err + } - gidField := v.FieldByName("Gid") + err = applyDqlMutations(ctx, db, dms) + if err != nil { + return 0, nil, false, err + } - if gidField.IsValid() && gidField.CanSet() && gidField.Kind() == reflect.Uint64 { - gidField.SetUint(gid) + gid, object, err = getByGid[T](ctx, n, gid) + if err != nil { + return 0, nil, false, err } - return gid, object, nil + return gid, object, wasFound, nil } func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { db.mutex.Lock() defer db.mutex.Unlock() + if len(ns) > 1 { + return 0, nil, fmt.Errorf("only one namespace is allowed") + } ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { return 0, nil, err @@ -104,6 +140,9 @@ func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { db.mutex.Lock() defer db.mutex.Unlock() + if len(ns) > 1 { + return 0, nil, fmt.Errorf("only one namespace is allowed") + } ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { return 0, nil, err @@ -125,7 +164,19 @@ func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, } if cf, ok := any(uniqueField).(ConstrainedField); ok { - return getByConstrainedField[T](ctx, n, cf) + uid, obj, err := getByConstrainedField[T](ctx, n, cf) + if err != nil { + return 0, nil, err + } + + dms := generateDeleteDqlMutations(n, uid) + + err = applyDqlMutations(ctx, db, dms) + if err != nil { + return 0, nil, err + } + + return uid, obj, nil } return 0, nil, fmt.Errorf("invalid unique field type") diff --git a/api_dql.go b/api_dql.go new file mode 100644 index 0000000..a2b0e65 --- /dev/null +++ b/api_dql.go @@ -0,0 +1,41 @@ +package modusdb + +import "fmt" + +type QueryFunc func() string + +const ( + objQuery = ` + { + obj(%s) { + uid + expand(_all_) { + uid + expand(_all_) + dgraph.type + } + dgraph.type + %s + } + } + ` + + funcUid = `func: uid(%d)` + funcEq = `func: eq(%s, %s)` +) + +func buildUidQuery(gid uint64) QueryFunc { + return func() string { + return fmt.Sprintf(funcUid, gid) + } +} + +func buildEqQuery(key, value any) QueryFunc { + return func() string { + return fmt.Sprintf(funcEq, key, value) + } +} + +func formatObjQuery(qf QueryFunc, extraFields string) string { + return fmt.Sprintf(objQuery, qf(), extraFields) +} diff --git a/api_helper.go b/api_helper.go deleted file mode 100644 index 0cda75b..0000000 --- a/api_helper.go +++ /dev/null @@ -1,301 +0,0 @@ -package modusdb - -import ( - "context" - "encoding/binary" - "encoding/json" - "fmt" - "reflect" - "time" - - "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/dgraph-io/dgraph/v24/dql" - "github.com/dgraph-io/dgraph/v24/protos/pb" - "github.com/dgraph-io/dgraph/v24/query" - "github.com/dgraph-io/dgraph/v24/schema" - "github.com/dgraph-io/dgraph/v24/worker" - "github.com/dgraph-io/dgraph/v24/x" - "github.com/twpayne/go-geom" - "github.com/twpayne/go-geom/encoding/wkb" -) - -func getPredicateName(typeName, fieldName string) string { - return fmt.Sprint(typeName, ".", fieldName) -} - -func addNamespace(ns uint64, pred string) string { - return x.NamespaceAttr(ns, pred) -} - -func valueToPosting_ValType(v any) (pb.Posting_ValType, error) { - switch v.(type) { - case string: - return pb.Posting_STRING, nil - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: - return pb.Posting_INT, nil - case bool: - return pb.Posting_BOOL, nil - case float32, float64: - return pb.Posting_FLOAT, nil - case []byte: - return pb.Posting_BINARY, nil - case time.Time: - return pb.Posting_DATETIME, nil - case geom.Point: - return pb.Posting_GEO, nil - case []float32, []float64: - return pb.Posting_VFLOAT, nil - default: - return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) - } -} - -func valueToValType(v any) (*api.Value, error) { - switch val := v.(type) { - case string: - return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil - case int: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int8: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int16: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int32: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int64: - return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil - case uint8: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint16: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint32: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case bool: - return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil - case float32: - return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil - case float64: - return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil - case []byte: - return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil - case time.Time: - bytes, err := val.MarshalBinary() - if err != nil { - return nil, err - } - return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil - case geom.Point: - bytes, err := wkb.Marshal(&val, binary.LittleEndian) - if err != nil { - return nil, err - } - return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil - case uint, uint64: - return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil - default: - return nil, fmt.Errorf("unsupported type %T", v) - } -} - -func generateCreateDqlMutationsAndSchema[T any](n *Namespace, object *T, - gid uint64) ([]*dql.Mutation, *schema.ParsedSchema, error) { - t := reflect.TypeOf(*object) - if t.Kind() != reflect.Struct { - return nil, nil, fmt.Errorf("expected struct, got %s", t.Kind()) - } - - jsonFields, dbFields, _, err := getFieldTags(t) - if err != nil { - return nil, nil, err - } - values := getFieldValues(object, jsonFields) - sch := &schema.ParsedSchema{} - - nquads := make([]*api.NQuad, 0) - for jsonName, value := range values { - if jsonName == "gid" { - continue - } - valType, err := valueToPosting_ValType(value) - if err != nil { - return nil, nil, err - } - u := &pb.SchemaUpdate{ - Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), - ValueType: valType, - } - if dbFields[jsonName] != nil && dbFields[jsonName].constraint == "unique" { - u.Directive = pb.SchemaUpdate_INDEX - u.Tokenizer = []string{"exact"} - } - sch.Preds = append(sch.Preds, u) - val, err := valueToValType(value) - if err != nil { - return nil, nil, err - } - nquad := &api.NQuad{ - Namespace: n.ID(), - Subject: fmt.Sprint(gid), - Predicate: getPredicateName(t.Name(), jsonName), - ObjectValue: val, - } - nquads = append(nquads, nquad) - } - sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: addNamespace(n.id, t.Name()), - Fields: sch.Preds, - }) - - val, err := valueToValType(t.Name()) - if err != nil { - return nil, nil, err - } - nquad := &api.NQuad{ - Namespace: n.ID(), - Subject: fmt.Sprint(gid), - Predicate: "dgraph.type", - ObjectValue: val, - } - nquads = append(nquads, nquad) - - dms := make([]*dql.Mutation, 0) - dms = append(dms, &dql.Mutation{ - Set: nquads, - }) - - return dms, sch, nil -} - -func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { - return []*dql.Mutation{{ - Del: []*api.NQuad{ - { - Namespace: n.ID(), - Subject: fmt.Sprint(gid), - Predicate: x.Star, - ObjectValue: &api.Value{ - Val: &api.Value_DefaultVal{DefaultVal: x.Star}, - }, - }, - }, - }} -} - -func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, *T, error) { - query := fmt.Sprintf(` - { - obj(func: uid(%d)) { - uid - expand(_all_) - dgraph.type - } - } - `, gid) - - return executeGet[T](ctx, n, query, nil) -} - -func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, *T, error) { - var obj T - - t := reflect.TypeOf(obj) - query := fmt.Sprintf(` - { - obj(func: eq(%s, %s)) { - uid - expand(_all_) - dgraph.type - } - } - `, getPredicateName(t.Name(), cf.Key), cf.Value) - - return executeGet[T](ctx, n, query, &cf) -} - -func executeGet[T any](ctx context.Context, n *Namespace, query string, cf *ConstrainedField) (uint64, *T, error) { - var obj T - - t := reflect.TypeOf(obj) - - jsonFields, dbTags, _, err := getFieldTags(t) - if err != nil { - return 0, nil, err - } - - if cf != nil && dbTags[cf.Key].constraint == "" { - return 0, nil, fmt.Errorf("constraint not defined for field %s", cf.Key) - } - - resp, err := n.queryWithLock(ctx, query) - if err != nil { - return 0, nil, err - } - - dynamicType := createDynamicStruct(t, jsonFields) - - dynamicInstance := reflect.New(dynamicType).Interface() - - var result struct { - Obj []any `json:"obj"` - } - - result.Obj = append(result.Obj, dynamicInstance) - - // Unmarshal the JSON response into the dynamic struct - if err := json.Unmarshal(resp.Json, &result); err != nil { - return 0, nil, err - } - - // Check if we have at least one object in the response - if len(result.Obj) == 0 { - return 0, nil, ErrNoObjFound - } - - // Map the dynamic struct to the final type T - finalObject := reflect.New(t).Interface() - gid, err := mapDynamicToFinal(result.Obj[0], finalObject) - if err != nil { - return 0, nil, err - } - - return gid, finalObject.(*T), nil -} - -func applyDqlMutations(ctx context.Context, db *DB, dms []*dql.Mutation) error { - edges, err := query.ToDirectedEdges(dms, nil) - if err != nil { - return err - } - - if !db.isOpen { - return ErrClosedDB - } - - startTs, err := db.z.nextTs() - if err != nil { - return err - } - commitTs, err := db.z.nextTs() - if err != nil { - return err - } - - m := &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Edges: edges, - } - m.Edges, err = query.ExpandEdges(ctx, m) - if err != nil { - return fmt.Errorf("error expanding edges: %w", err) - } - - p := &pb.Proposal{Mutations: m, StartTs: startTs} - if err := worker.ApplyMutations(ctx, p); err != nil { - return err - } - - return worker.ApplyCommited(ctx, &pb.OracleDelta{ - Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, - }) -} diff --git a/api_mutate_helper.go b/api_mutate_helper.go new file mode 100644 index 0000000..7957e0a --- /dev/null +++ b/api_mutate_helper.go @@ -0,0 +1,240 @@ +package modusdb + +import ( + "context" + "fmt" + "reflect" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/dql" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/query" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/worker" + "github.com/dgraph-io/dgraph/v24/x" +) + +func generateCreateDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, + gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { + t := reflect.TypeOf(object) + if t.Kind() != reflect.Struct { + return fmt.Errorf("expected struct, got %s", t.Kind()) + } + + fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, err := getFieldTags(t) + if err != nil { + return err + } + jsonTagToValue := getJsonTagToValues(object, fieldToJsonTags) + + nquads := make([]*api.NQuad, 0) + uniqueConstraintFound := false + for jsonName, value := range jsonTagToValue { + if jsonToReverseEdgeTags[jsonName] != "" { + continue + } + if jsonName == "gid" { + uniqueConstraintFound = true + continue + } + var val *api.Value + var valType pb.Posting_ValType + + reflectValueType := reflect.TypeOf(value) + var nquad *api.NQuad + + if reflectValueType.Kind() == reflect.Struct { + value = reflect.ValueOf(value).Interface() + newGid, err := getUidOrMutate(ctx, n.db, n, value) + if err != nil { + return err + } + value = newGid + } else if reflectValueType.Kind() == reflect.Pointer { + // dereference the pointer + reflectValueType = reflectValueType.Elem() + if reflectValueType.Kind() == reflect.Struct { + // convert value to pointer, and then dereference + value = reflect.ValueOf(value).Elem().Interface() + newGid, err := getUidOrMutate(ctx, n.db, n, value) + if err != nil { + return err + } + value = newGid + } + } + valType, err = valueToPosting_ValType(value) + if err != nil { + return err + } + val, err = valueToApiVal(value) + if err != nil { + return err + } + + nquad = &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: getPredicateName(t.Name(), jsonName), + } + + if valType == pb.Posting_UID { + nquad.ObjectId = fmt.Sprint(value) + } else { + nquad.ObjectValue = val + } + + u := &pb.SchemaUpdate{ + Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), + ValueType: valType, + } + if jsonToDbTags[jsonName] != nil { + constraint := jsonToDbTags[jsonName].constraint + if constraint == "unique" || constraint == "term" { + uniqueConstraintFound = true + u.Directive = pb.SchemaUpdate_INDEX + if constraint == "unique" { + u.Tokenizer = []string{"exact"} + } else { + u.Tokenizer = []string{"term"} + } + } + } + + sch.Preds = append(sch.Preds, u) + nquads = append(nquads, nquad) + } + if !uniqueConstraintFound { + return fmt.Errorf(NoUniqueConstr, t.Name()) + } + sch.Types = append(sch.Types, &pb.TypeUpdate{ + TypeName: addNamespace(n.id, t.Name()), + Fields: sch.Preds, + }) + + val, err := valueToApiVal(t.Name()) + if err != nil { + return err + } + typeNquad := &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: "dgraph.type", + ObjectValue: val, + } + nquads = append(nquads, typeNquad) + + *dms = append(*dms, &dql.Mutation{ + Set: nquads, + }) + + return nil +} + +func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { + return []*dql.Mutation{{ + Del: []*api.NQuad{ + { + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: x.Star, + ObjectValue: &api.Value{ + Val: &api.Value_DefaultVal{DefaultVal: x.Star}, + }, + }, + }, + }} +} + +func applyDqlMutations(ctx context.Context, db *DB, dms []*dql.Mutation) error { + edges, err := query.ToDirectedEdges(dms, nil) + if err != nil { + return err + } + + if !db.isOpen { + return ErrClosedDB + } + + startTs, err := db.z.nextTs() + if err != nil { + return err + } + commitTs, err := db.z.nextTs() + if err != nil { + return err + } + + m := &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Edges: edges, + } + m.Edges, err = query.ExpandEdges(ctx, m) + if err != nil { + return fmt.Errorf("error expanding edges: %w", err) + } + + p := &pb.Proposal{Mutations: m, StartTs: startTs} + if err := worker.ApplyMutations(ctx, p); err != nil { + return err + } + + return worker.ApplyCommited(ctx, &pb.OracleDelta{ + Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, + }) +} + +func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) (uint64, error) { + gid, cf, err := getUniqueConstraint[T](object) + if err != nil { + return 0, err + } + + dms := make([]*dql.Mutation, 0) + sch := &schema.ParsedSchema{} + err = generateCreateDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) + if err != nil { + return 0, err + } + + err = n.alterSchemaWithParsed(ctx, sch) + if err != nil { + return 0, err + } + if gid != 0 { + gid, _, err = getByGidWithObject[T](ctx, n, gid, object) + if err != nil && err != ErrNoObjFound { + return 0, err + } + if err == nil { + return gid, nil + } + } else if cf != nil { + gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, object) + if err != nil && err != ErrNoObjFound { + return 0, err + } + if err == nil { + return gid, nil + } + } + + gid, err = db.z.nextUID() + if err != nil { + return 0, err + } + + dms = make([]*dql.Mutation, 0) + err = generateCreateDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) + if err != nil { + return 0, err + } + + err = applyDqlMutations(ctx, db, dms) + if err != nil { + return 0, err + } + + return gid, nil +} diff --git a/api_query_helper.go b/api_query_helper.go new file mode 100644 index 0000000..e62d8f3 --- /dev/null +++ b/api_query_helper.go @@ -0,0 +1,123 @@ +package modusdb + +import ( + "context" + "encoding/json" + "fmt" + "reflect" +) + +func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, *T, error) { + return executeGet[T](ctx, n, gid) +} + +func getByGidWithObject[T any](ctx context.Context, n *Namespace, gid uint64, obj T) (uint64, *T, error) { + return executeGetWithObject[T](ctx, n, obj, false, gid) +} + +func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, *T, error) { + return executeGet[T](ctx, n, cf) +} + +func getByConstrainedFieldWithObject[T any](ctx context.Context, n *Namespace, + cf ConstrainedField, obj T) (uint64, *T, error) { + + return executeGetWithObject[T](ctx, n, obj, false, cf) +} + +func executeGet[T any, R UniqueField](ctx context.Context, n *Namespace, args ...R) (uint64, *T, error) { + if len(args) != 1 { + return 0, nil, fmt.Errorf("expected 1 argument, got %d", len(args)) + } + + var obj T + + return executeGetWithObject(ctx, n, obj, true, args...) +} + +func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespace, + obj T, withReverse bool, args ...R) (uint64, *T, error) { + t := reflect.TypeOf(obj) + + fieldToJsonTags, jsonToDbTag, jsonToReverseEdgeTags, err := getFieldTags(t) + if err != nil { + return 0, nil, err + } + readFromQuery := "" + if withReverse { + for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { + readFromQuery += fmt.Sprintf(` + %s: ~%s { + uid + expand(_all_) + dgraph.type + } + `, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) + } + } + + var cf ConstrainedField + var query string + gid, ok := any(args[0]).(uint64) + if ok { + query = formatObjQuery(buildUidQuery(gid), readFromQuery) + } else if cf, ok = any(args[0]).(ConstrainedField); ok { + query = formatObjQuery(buildEqQuery(getPredicateName(t.Name(), cf.Key), cf.Value), readFromQuery) + } else { + return 0, nil, fmt.Errorf("invalid unique field type") + } + + if jsonToDbTag[cf.Key] != nil && jsonToDbTag[cf.Key].constraint == "" { + return 0, nil, fmt.Errorf("constraint not defined for field %s", cf.Key) + } + + resp, err := n.queryWithLock(ctx, query) + if err != nil { + return 0, nil, err + } + + dynamicType := createDynamicStruct(t, fieldToJsonTags, 1) + + dynamicInstance := reflect.New(dynamicType).Interface() + + var result struct { + Obj []any `json:"obj"` + } + + result.Obj = append(result.Obj, dynamicInstance) + + // Unmarshal the JSON response into the dynamic struct + if err := json.Unmarshal(resp.Json, &result); err != nil { + return 0, nil, err + } + + // Check if we have at least one object in the response + if len(result.Obj) == 0 { + return 0, nil, ErrNoObjFound + } + + // Map the dynamic struct to the final type T + finalObject := reflect.New(t).Interface() + gid, err = mapDynamicToFinal(result.Obj[0], finalObject) + if err != nil { + return 0, nil, err + } + + // Convert to *interface{} then to *T + if ifacePtr, ok := finalObject.(*interface{}); ok { + if typedPtr, ok := (*ifacePtr).(*T); ok { + return gid, typedPtr, nil + } + } + + // If conversion fails, try direct conversion + if typedPtr, ok := finalObject.(*T); ok { + return gid, typedPtr, nil + } + + if dirType, ok := finalObject.(T); ok { + return gid, &dirType, nil + } + + return 0, nil, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) +} diff --git a/api_reflect.go b/api_reflect.go index 4c84813..f74cd53 100644 --- a/api_reflect.go +++ b/api_reflect.go @@ -11,12 +11,12 @@ type dbTag struct { constraint string } -func getFieldTags(t reflect.Type) (jsonTags map[string]string, jsonToDbTags map[string]*dbTag, - reverseEdgeTags map[string]string, err error) { +func getFieldTags(t reflect.Type) (fieldToJsonTags map[string]string, + jsonToDbTags map[string]*dbTag, jsonToReverseEdgeTags map[string]string, err error) { - jsonTags = make(map[string]string) + fieldToJsonTags = make(map[string]string) jsonToDbTags = make(map[string]*dbTag) - reverseEdgeTags = make(map[string]string) + jsonToReverseEdgeTags = make(map[string]string) for i := 0; i < t.NumField(); i++ { field := t.Field(i) jsonTag := field.Tag.Get("json") @@ -24,7 +24,7 @@ func getFieldTags(t reflect.Type) (jsonTags map[string]string, jsonToDbTags map[ return nil, nil, nil, fmt.Errorf("field %s has no json tag", field.Name) } jsonName := strings.Split(jsonTag, ",")[0] - jsonTags[field.Name] = jsonName + fieldToJsonTags[field.Name] = jsonName reverseEdgeTag := field.Tag.Get("readFrom") if reverseEdgeTag != "" { @@ -35,7 +35,7 @@ func getFieldTags(t reflect.Type) (jsonTags map[string]string, jsonToDbTags map[ } t := strings.Split(typeAndField[0], "=")[1] f := strings.Split(typeAndField[1], "=")[1] - reverseEdgeTags[field.Name] = getPredicateName(t, f) + jsonToReverseEdgeTags[jsonName] = getPredicateName(t, f) } dbConstraintsTag := field.Tag.Get("db") @@ -50,13 +50,16 @@ func getFieldTags(t reflect.Type) (jsonTags map[string]string, jsonToDbTags map[ } } } - return jsonTags, jsonToDbTags, reverseEdgeTags, nil + return fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, nil } -func getFieldValues(object any, jsonFields map[string]string) map[string]any { +func getJsonTagToValues(object any, fieldToJsonTags map[string]string) map[string]any { values := make(map[string]any) - v := reflect.ValueOf(object).Elem() - for fieldName, jsonName := range jsonFields { + v := reflect.ValueOf(object) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + for fieldName, jsonName := range fieldToJsonTags { fieldValue := v.FieldByName(fieldName) values[jsonName] = fieldValue.Interface() @@ -64,16 +67,38 @@ func getFieldValues(object any, jsonFields map[string]string) map[string]any { return values } -func createDynamicStruct(t reflect.Type, jsonFields map[string]string) reflect.Type { - fields := make([]reflect.StructField, 0, len(jsonFields)) - for fieldName, jsonName := range jsonFields { +func createDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, depth int) reflect.Type { + fields := make([]reflect.StructField, 0, len(fieldToJsonTags)) + for fieldName, jsonName := range fieldToJsonTags { field, _ := t.FieldByName(fieldName) if fieldName != "Gid" { - fields = append(fields, reflect.StructField{ - Name: field.Name, - Type: field.Type, - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), - }) + if field.Type.Kind() == reflect.Struct { + if depth <= 2 { + nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type) + nestedType := createDynamicStruct(field.Type, nestedFieldToJsonTags, depth+1) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: nestedType, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } + } else if field.Type.Kind() == reflect.Ptr && + field.Type.Elem().Kind() == reflect.Struct { + nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type.Elem()) + nestedType := createDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: reflect.PointerTo(nestedType), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } else { + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: field.Type, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } + } } fields = append(fields, reflect.StructField{ @@ -95,30 +120,80 @@ func mapDynamicToFinal(dynamic any, final any) (uint64, error) { gid := uint64(0) for i := 0; i < vDynamic.NumField(); i++ { - field := vDynamic.Type().Field(i) - value := vDynamic.Field(i) + + dynamicField := vDynamic.Type().Field(i) + dynamicFieldType := dynamicField.Type + dynamicValue := vDynamic.Field(i) var finalField reflect.Value - if field.Name == "Uid" { + if dynamicField.Name == "Uid" { finalField = vFinal.FieldByName("Gid") - gidStr := value.String() + gidStr := dynamicValue.String() gid, _ = strconv.ParseUint(gidStr, 0, 64) - } else if field.Name == "DgraphType" { - fieldArr := value.Interface().([]string) + } else if dynamicField.Name == "DgraphType" { + fieldArr := dynamicValue.Interface().([]string) if len(fieldArr) == 0 { return 0, ErrNoObjFound } } else { - finalField = vFinal.FieldByName(field.Name) + finalField = vFinal.FieldByName(dynamicField.Name) } - if finalField.IsValid() && finalField.CanSet() { - // if field name is uid, convert it to uint64 - if field.Name == "Uid" { - finalField.SetUint(gid) - } else { - finalField.Set(value) + if dynamicFieldType.Kind() == reflect.Struct { + _, err := mapDynamicToFinal(dynamicValue.Addr().Interface(), finalField.Addr().Interface()) + if err != nil { + return 0, err + } + } else if dynamicFieldType.Kind() == reflect.Ptr && + dynamicFieldType.Elem().Kind() == reflect.Struct { + // if field is a pointer, find if the underlying is a struct + _, err := mapDynamicToFinal(dynamicValue.Interface(), finalField.Interface()) + if err != nil { + return 0, err + } + + } else { + if finalField.IsValid() && finalField.CanSet() { + // if field name is uid, convert it to uint64 + if dynamicField.Name == "Uid" { + finalField.SetUint(gid) + } else { + finalField.Set(dynamicValue) + } } } } return gid, nil } + +func getUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { + t := reflect.TypeOf(object) + fieldToJsonTags, jsonToDbTags, _, err := getFieldTags(t) + if err != nil { + return 0, nil, err + } + jsonTagToValue := getJsonTagToValues(object, fieldToJsonTags) + + for jsonName, value := range jsonTagToValue { + if jsonName == "gid" { + gid, ok := value.(uint64) + if !ok { + continue + } + if gid != 0 { + return gid, nil, nil + } + } + if jsonToDbTags[jsonName] != nil && jsonToDbTags[jsonName].constraint == "unique" { + // check if value is zero or nil + if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { + continue + } + return 0, &ConstrainedField{ + Key: jsonName, + Value: value, + }, nil + } + } + + return 0, nil, fmt.Errorf(NoUniqueConstr, t.Name()) +} diff --git a/api_test.go b/api_test.go index 5b1c6bc..1d3293c 100644 --- a/api_test.go +++ b/api_test.go @@ -2,6 +2,7 @@ package modusdb_test import ( "context" + "fmt" "testing" "github.com/stretchr/testify/require" @@ -16,6 +17,52 @@ type User struct { ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` } +func TestFirstTimeUser(t *testing.T) { + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + gid, user, err := modusdb.Create(db, &User{ + Name: "A", + Age: 10, + ClerkId: "123", + }) + + require.NoError(t, err) + require.Equal(t, user.Gid, gid) + require.Equal(t, "A", user.Name) + require.Equal(t, 10, user.Age) + require.Equal(t, "123", user.ClerkId) + + gid, queriedUser, err := modusdb.Get[User](db, gid) + + require.NoError(t, err) + require.Equal(t, queriedUser.Gid, gid) + require.Equal(t, 10, queriedUser.Age) + require.Equal(t, "A", queriedUser.Name) + require.Equal(t, "123", queriedUser.ClerkId) + + gid, queriedUser2, err := modusdb.Get[User](db, modusdb.ConstrainedField{ + Key: "clerk_id", + Value: "123", + }) + + require.NoError(t, err) + require.Equal(t, queriedUser.Gid, gid) + require.Equal(t, 10, queriedUser2.Age) + require.Equal(t, "A", queriedUser2.Name) + require.Equal(t, "123", queriedUser2.ClerkId) + + _, _, err = modusdb.Delete[User](db, gid) + require.NoError(t, err) + + _, queriedUser3, err := modusdb.Get[User](db, gid) + require.Error(t, err) + require.Equal(t, "no object found", err.Error()) + require.Nil(t, queriedUser3) + +} + func TestCreateApi(t *testing.T) { ctx := context.Background() db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) @@ -33,12 +80,11 @@ func TestCreateApi(t *testing.T) { ClerkId: "123", } - gid, _, err := modusdb.Create(db, user, db1.ID()) + gid, user, err := modusdb.Create(db, user, db1.ID()) require.NoError(t, err) require.Equal(t, "B", user.Name) - require.Equal(t, uint64(2), gid) - require.Equal(t, uint64(2), user.Gid) + require.Equal(t, user.Gid, gid) query := `{ me(func: has(User.name)) { @@ -118,8 +164,7 @@ func TestGetApi(t *testing.T) { gid, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) require.NoError(t, err) - require.Equal(t, uint64(2), gid) - require.Equal(t, uint64(2), queriedUser.Gid) + require.Equal(t, queriedUser.Gid, gid) require.Equal(t, 20, queriedUser.Age) require.Equal(t, "B", queriedUser.Name) require.Equal(t, "123", queriedUser.ClerkId) @@ -151,8 +196,7 @@ func TestGetApiWithConstrainedField(t *testing.T) { }, db1.ID()) require.NoError(t, err) - require.Equal(t, uint64(2), gid) - require.Equal(t, uint64(2), queriedUser.Gid) + require.Equal(t, queriedUser.Gid, gid) require.Equal(t, 20, queriedUser.Age) require.Equal(t, "B", queriedUser.Name) require.Equal(t, "123", queriedUser.ClerkId) @@ -194,3 +238,277 @@ func TestDeleteApi(t *testing.T) { require.Equal(t, "no object found", err.Error()) require.Nil(t, queriedUser) } + +func TestUpsertApi(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + user := &User{ + Name: "B", + Age: 20, + ClerkId: "123", + } + + gid, user, _, err := modusdb.Upsert(db, user, db1.ID()) + require.NoError(t, err) + require.Equal(t, user.Gid, gid) + + user.Age = 21 + gid, _, _, err = modusdb.Upsert(db, user, db1.ID()) + require.NoError(t, err) + require.Equal(t, user.Gid, gid) + + _, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) + require.NoError(t, err) + require.Equal(t, user.Gid, queriedUser.Gid) + require.Equal(t, 21, queriedUser.Age) + require.Equal(t, "B", queriedUser.Name) + require.Equal(t, "123", queriedUser.ClerkId) +} + +type Project struct { + Gid uint64 `json:"gid,omitempty"` + Name string `json:"name,omitempty"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + // Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` +} + +type Branch struct { + Gid uint64 `json:"gid,omitempty"` + Name string `json:"name,omitempty"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + Proj Project `json:"proj,omitempty"` +} + +func TestNestedObjectMutation(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + branch := &Branch{ + Name: "B", + ClerkId: "123", + Proj: Project{ + Name: "P", + ClerkId: "456", + }, + } + + gid, branch, err := modusdb.Create(db, branch, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "B", branch.Name) + require.Equal(t, branch.Gid, gid) + require.NotEqual(t, uint64(0), branch.Proj.Gid) + require.Equal(t, "P", branch.Proj.Name) + + query := `{ + me(func: has(Branch.name)) { + uid + Branch.name + Branch.clerk_id + Branch.proj { + uid + Project.name + Project.clerk_id + } + } + }` + resp, err := db1.Query(ctx, query) + require.NoError(t, err) + require.JSONEq(t, + `{"me":[{"uid":"0x2","Branch.name":"B","Branch.clerk_id":"123","Branch.proj": + {"uid":"0x3","Project.name":"P","Project.clerk_id":"456"}}]}`, + string(resp.GetJson())) + + gid, queriedBranch, err := modusdb.Get[Branch](db, gid, db1.ID()) + require.NoError(t, err) + require.Equal(t, queriedBranch.Gid, gid) + require.Equal(t, "B", queriedBranch.Name) + +} + +func TestLinkingObjectsByConstrainedFields(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + projGid, project, err := modusdb.Create(db, &Project{ + Name: "P", + ClerkId: "456", + }, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "P", project.Name) + require.Equal(t, project.Gid, projGid) + + branch := &Branch{ + Name: "B", + ClerkId: "123", + Proj: Project{ + Name: "P", + ClerkId: "456", + }, + } + + gid, branch, err := modusdb.Create(db, branch, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "B", branch.Name) + require.Equal(t, branch.Gid, gid) + require.Equal(t, projGid, branch.Proj.Gid) + require.Equal(t, "P", branch.Proj.Name) + + query := `{ + me(func: has(Branch.name)) { + uid + Branch.name + Branch.clerk_id + Branch.proj { + uid + Project.name + Project.clerk_id + } + } + }` + resp, err := db1.Query(ctx, query) + require.NoError(t, err) + require.JSONEq(t, + `{"me":[{"uid":"0x3","Branch.name":"B","Branch.clerk_id":"123","Branch.proj": + {"uid":"0x2","Project.name":"P","Project.clerk_id":"456"}}]}`, + string(resp.GetJson())) + + gid, queriedBranch, err := modusdb.Get[Branch](db, gid, db1.ID()) + require.NoError(t, err) + require.Equal(t, queriedBranch.Gid, gid) + require.Equal(t, "B", queriedBranch.Name) + +} + +func TestLinkingObjectsByGid(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + projGid, project, err := modusdb.Create(db, &Project{ + Name: "P", + ClerkId: "456", + }, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "P", project.Name) + require.Equal(t, project.Gid, projGid) + + branch := &Branch{ + Name: "B", + ClerkId: "123", + Proj: Project{ + Gid: projGid, + }, + } + + gid, branch, err := modusdb.Create(db, branch, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "B", branch.Name) + require.Equal(t, branch.Gid, gid) + require.Equal(t, projGid, branch.Proj.Gid) + require.Equal(t, "P", branch.Proj.Name) + + query := `{ + me(func: has(Branch.name)) { + uid + Branch.name + Branch.clerk_id + Branch.proj { + uid + Project.name + Project.clerk_id + } + } + }` + resp, err := db1.Query(ctx, query) + require.NoError(t, err) + require.JSONEq(t, + `{"me":[{"uid":"0x3","Branch.name":"B","Branch.clerk_id":"123", + "Branch.proj":{"uid":"0x2","Project.name":"P","Project.clerk_id":"456"}}]}`, + string(resp.GetJson())) + + gid, queriedBranch, err := modusdb.Get[Branch](db, gid, db1.ID()) + require.NoError(t, err) + require.Equal(t, queriedBranch.Gid, gid) + require.Equal(t, "B", queriedBranch.Name) + +} + +type BadProject struct { + Name string `json:"name,omitempty"` + ClerkId string `json:"clerk_id,omitempty"` +} + +type BadBranch struct { + Gid uint64 `json:"gid,omitempty"` + Name string `json:"name,omitempty"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + Proj BadProject `json:"proj,omitempty"` +} + +func TestNestedObjectMutationWithBadType(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + branch := &BadBranch{ + Name: "B", + ClerkId: "123", + Proj: BadProject{ + Name: "P", + ClerkId: "456", + }, + } + + _, _, err = modusdb.Create(db, branch, db1.ID()) + require.Error(t, err) + require.Equal(t, fmt.Sprintf(modusdb.NoUniqueConstr, "BadProject"), err.Error()) + + proj := &BadProject{ + Name: "P", + ClerkId: "456", + } + + _, _, err = modusdb.Create(db, proj, db1.ID()) + require.Error(t, err) + require.Equal(t, fmt.Sprintf(modusdb.NoUniqueConstr, "BadProject"), err.Error()) + +} diff --git a/api_types.go b/api_types.go index 9cea571..860edda 100644 --- a/api_types.go +++ b/api_types.go @@ -1,9 +1,21 @@ package modusdb -import "fmt" +import ( + "context" + "encoding/binary" + "fmt" + "time" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/x" + "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/wkb" +) var ( - ErrNoObjFound = fmt.Errorf("no object found") + ErrNoObjFound = fmt.Errorf("no object found") + NoUniqueConstr = "unique constraint not defined for any field on type %s" ) type UniqueField interface { @@ -13,3 +25,108 @@ type ConstrainedField struct { Key string Value any } + +type ModusDbOption func(*modusDbOptions) + +type modusDbOptions struct { + namespace uint64 +} + +func WithNamespace(namespace uint64) ModusDbOption { + return func(o *modusDbOptions) { + o.namespace = namespace + } +} + +func getDefaultNamespace(db *DB, ns ...uint64) (context.Context, *Namespace, error) { + dbOpts := &modusDbOptions{ + namespace: db.defaultNamespace.ID(), + } + for _, ns := range ns { + WithNamespace(ns)(dbOpts) + } + + n, err := db.getNamespaceWithLock(dbOpts.namespace) + if err != nil { + return nil, nil, err + } + + ctx := context.Background() + ctx = x.AttachNamespace(ctx, n.ID()) + + return ctx, n, nil +} + +func valueToPosting_ValType(v any) (pb.Posting_ValType, error) { + switch v.(type) { + case string: + return pb.Posting_STRING, nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32: + return pb.Posting_INT, nil + case uint64: + return pb.Posting_UID, nil + case bool: + return pb.Posting_BOOL, nil + case float32, float64: + return pb.Posting_FLOAT, nil + case []byte: + return pb.Posting_BINARY, nil + case time.Time: + return pb.Posting_DATETIME, nil + case geom.Point: + return pb.Posting_GEO, nil + case []float32, []float64: + return pb.Posting_VFLOAT, nil + default: + return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) + } +} + +func valueToApiVal(v any) (*api.Value, error) { + switch val := v.(type) { + case string: + return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil + case int: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int64: + return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil + case uint8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint64: + return &api.Value{Val: &api.Value_UidVal{UidVal: val}}, nil + case bool: + return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil + case float32: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil + case float64: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil + case []byte: + return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil + case time.Time: + bytes, err := val.MarshalBinary() + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil + case geom.Point: + bytes, err := wkb.Marshal(&val, binary.LittleEndian) + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil + case uint: + return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil + default: + return nil, fmt.Errorf("unsupported type %T", v) + } +} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..133ee1d --- /dev/null +++ b/utils.go @@ -0,0 +1,15 @@ +package modusdb + +import ( + "fmt" + + "github.com/dgraph-io/dgraph/v24/x" +) + +func getPredicateName(typeName, fieldName string) string { + return fmt.Sprint(typeName, ".", fieldName) +} + +func addNamespace(ns uint64, pred string) string { + return x.NamespaceAttr(ns, pred) +} From 84b781b06f3903a881c288882b92ff21f5442ee6 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 2 Jan 2025 08:19:53 -0800 Subject: [PATCH 10/23] support unstructured data in modusdb (#38) --- api.go | 77 +++++++------ api_dql.go | 179 ++++++++++++++++++++++++++++- api_mutate_helper.go | 47 ++++++-- api_query_helper.go | 137 +++++++++++++++++----- api_reflect.go | 6 +- api_test.go | 268 ++++++++++++++++++++++++++++++++++++++++--- api_types.go | 108 +++++++++++++++++ 7 files changed, 733 insertions(+), 89 deletions(-) diff --git a/api.go b/api.go index 7fcdccd..e329198 100644 --- a/api.go +++ b/api.go @@ -7,7 +7,7 @@ import ( "github.com/dgraph-io/dgraph/v24/schema" ) -func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { +func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { db.mutex.Lock() defer db.mutex.Unlock() if len(ns) > 1 { @@ -25,7 +25,7 @@ func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateCreateDqlMutationsAndSchema[T](ctx, n, *object, gid, &dms, sch) + err = generateCreateDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) if err != nil { return 0, object, err } @@ -43,7 +43,7 @@ func Create[T any](db *DB, object *T, ns ...uint64) (uint64, *T, error) { return getByGid[T](ctx, n, gid) } -func Upsert[T any](db *DB, object *T, ns ...uint64) (uint64, *T, bool, error) { +func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { var wasFound bool db.mutex.Lock() @@ -51,80 +51,78 @@ func Upsert[T any](db *DB, object *T, ns ...uint64) (uint64, *T, bool, error) { if len(ns) > 1 { return 0, object, false, fmt.Errorf("only one namespace is allowed") } - if object == nil { - return 0, nil, false, fmt.Errorf("object is nil") - } ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { return 0, object, false, err } - gid, cf, err := getUniqueConstraint[T](*object) + gid, cf, err := getUniqueConstraint[T](object) if err != nil { - return 0, nil, false, err + return 0, object, false, err } dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateCreateDqlMutationsAndSchema[T](ctx, n, *object, gid, &dms, sch) + err = generateCreateDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) if err != nil { - return 0, nil, false, err + return 0, object, false, err } err = n.alterSchemaWithParsed(ctx, sch) if err != nil { - return 0, nil, false, err + return 0, object, false, err } if gid != 0 { - gid, _, err = getByGidWithObject[T](ctx, n, gid, *object) + gid, _, err = getByGidWithObject[T](ctx, n, gid, object) if err != nil && err != ErrNoObjFound { - return 0, nil, false, err + return 0, object, false, err } wasFound = err == nil } else if cf != nil { - gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, *object) + gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, object) if err != nil && err != ErrNoObjFound { - return 0, nil, false, err + return 0, object, false, err } wasFound = err == nil } if gid == 0 { gid, err = db.z.nextUID() if err != nil { - return 0, nil, false, err + return 0, object, false, err } } dms = make([]*dql.Mutation, 0) - err = generateCreateDqlMutationsAndSchema[T](ctx, n, *object, gid, &dms, sch) + err = generateCreateDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) if err != nil { - return 0, nil, false, err + return 0, object, false, err } err = applyDqlMutations(ctx, db, dms) if err != nil { - return 0, nil, false, err + return 0, object, false, err } gid, object, err = getByGid[T](ctx, n, gid) if err != nil { - return 0, nil, false, err + return 0, object, false, err } return gid, object, wasFound, nil } -func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { +func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, T, error) { db.mutex.Lock() defer db.mutex.Unlock() + var obj T if len(ns) > 1 { - return 0, nil, fmt.Errorf("only one namespace is allowed") + return 0, obj, fmt.Errorf("only one namespace is allowed") } ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { - return 0, nil, err + return 0, obj, err } if uid, ok := any(uniqueField).(uint64); ok { return getByGid[T](ctx, n, uid) @@ -134,30 +132,45 @@ func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, return getByConstrainedField[T](ctx, n, cf) } - return 0, nil, fmt.Errorf("invalid unique field type") + return 0, obj, fmt.Errorf("invalid unique field type") +} + +func Query[T any](db *DB, queryParams QueryParams, ns ...uint64) ([]uint64, []T, error) { + db.mutex.Lock() + defer db.mutex.Unlock() + if len(ns) > 1 { + return nil, nil, fmt.Errorf("only one namespace is allowed") + } + ctx, n, err := getDefaultNamespace(db, ns...) + if err != nil { + return nil, nil, err + } + + return executeQuery[T](ctx, n, queryParams, false) } -func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, *T, error) { +func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, T, error) { db.mutex.Lock() defer db.mutex.Unlock() + var zeroObj T if len(ns) > 1 { - return 0, nil, fmt.Errorf("only one namespace is allowed") + return 0, zeroObj, fmt.Errorf("only one namespace is allowed") } ctx, n, err := getDefaultNamespace(db, ns...) if err != nil { - return 0, nil, err + return 0, zeroObj, err } if uid, ok := any(uniqueField).(uint64); ok { uid, obj, err := getByGid[T](ctx, n, uid) if err != nil { - return 0, nil, err + return 0, zeroObj, err } dms := generateDeleteDqlMutations(n, uid) err = applyDqlMutations(ctx, db, dms) if err != nil { - return 0, nil, err + return 0, zeroObj, err } return uid, obj, nil @@ -166,18 +179,18 @@ func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, if cf, ok := any(uniqueField).(ConstrainedField); ok { uid, obj, err := getByConstrainedField[T](ctx, n, cf) if err != nil { - return 0, nil, err + return 0, zeroObj, err } dms := generateDeleteDqlMutations(n, uid) err = applyDqlMutations(ctx, db, dms) if err != nil { - return 0, nil, err + return 0, zeroObj, err } return uid, obj, nil } - return 0, nil, fmt.Errorf("invalid unique field type") + return 0, zeroObj, fmt.Errorf("invalid unique field type") } diff --git a/api_dql.go b/api_dql.go index a2b0e65..b20ecf0 100644 --- a/api_dql.go +++ b/api_dql.go @@ -1,13 +1,17 @@ package modusdb -import "fmt" +import ( + "fmt" + "strconv" + "strings" +) type QueryFunc func() string const ( objQuery = ` { - obj(%s) { + obj(func: %s) { uid expand(_all_) { uid @@ -20,8 +24,33 @@ const ( } ` - funcUid = `func: uid(%d)` - funcEq = `func: eq(%s, %s)` + objsQuery = ` + { + objs(func: type("%s")%s) @filter(%s) { + uid + expand(_all_) { + uid + expand(_all_) + dgraph.type + } + dgraph.type + %s + } + } + ` + + funcUid = `uid(%d)` + funcEq = `eq(%s, %s)` + funcSimilarTo = `similar_to(%s, %d, "[%s]")` + funcAllOfTerms = `allofterms(%s, "%s")` + funcAnyOfTerms = `anyofterms(%s, "%s")` + funcAllOfText = `alloftext(%s, "%s")` + funcAnyOfText = `anyoftext(%s, "%s")` + funcRegExp = `regexp(%s, /%s/)` + funcLe = `le(%s, %s)` + funcGe = `ge(%s, %s)` + funcGt = `gt(%s, %s)` + funcLt = `lt(%s, %s)` ) func buildUidQuery(gid uint64) QueryFunc { @@ -30,12 +59,152 @@ func buildUidQuery(gid uint64) QueryFunc { } } -func buildEqQuery(key, value any) QueryFunc { +func buildEqQuery(key string, value any) QueryFunc { return func() string { return fmt.Sprintf(funcEq, key, value) } } +func buildSimilarToQuery(indexAttr string, topK int64, vec []float32) QueryFunc { + vecStrArr := make([]string, len(vec)) + for i := range vec { + vecStrArr[i] = strconv.FormatFloat(float64(vec[i]), 'f', -1, 32) + } + vecStr := strings.Join(vecStrArr, ",") + return func() string { + return fmt.Sprintf(funcSimilarTo, indexAttr, topK, vecStr) + } +} + +func buildAllOfTermsQuery(attr string, terms string) QueryFunc { + return func() string { + return fmt.Sprintf(funcAllOfTerms, attr, terms) + } +} + +func buildAnyOfTermsQuery(attr string, terms string) QueryFunc { + return func() string { + return fmt.Sprintf(funcAnyOfTerms, attr, terms) + } +} + +func buildAllOfTextQuery(attr, text string) QueryFunc { + return func() string { + return fmt.Sprintf(funcAllOfText, attr, text) + } +} + +func buildAnyOfTextQuery(attr, text string) QueryFunc { + return func() string { + return fmt.Sprintf(funcAnyOfText, attr, text) + } +} + +func buildRegExpQuery(attr, pattern string) QueryFunc { + return func() string { + return fmt.Sprintf(funcRegExp, attr, pattern) + } +} + +func buildLeQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(funcLe, attr, value) + } +} + +func buildGeQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(funcGe, attr, value) + } +} + +func buildGtQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(funcGt, attr, value) + } +} + +func buildLtQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(funcLt, attr, value) + } +} + +func And(qfs ...QueryFunc) QueryFunc { + return func() string { + qs := make([]string, len(qfs)) + for i, qf := range qfs { + qs[i] = qf() + } + return strings.Join(qs, " AND ") + } +} + +func Or(qfs ...QueryFunc) QueryFunc { + return func() string { + qs := make([]string, len(qfs)) + for i, qf := range qfs { + qs[i] = qf() + } + return strings.Join(qs, " OR ") + } +} + +func Not(qf QueryFunc) QueryFunc { + return func() string { + return "NOT " + qf() + } +} + func formatObjQuery(qf QueryFunc, extraFields string) string { return fmt.Sprintf(objQuery, qf(), extraFields) } + +func formatObjsQuery(typeName string, qf QueryFunc, paginationAndSorting string, extraFields string) string { + return fmt.Sprintf(objsQuery, typeName, paginationAndSorting, qf(), extraFields) +} + +// Helper function to combine multiple filters +func filtersToQueryFunc(typeName string, filter Filter) QueryFunc { + return filterToQueryFunc(typeName, filter) +} + +func paginationToQueryString(p Pagination) string { + paginationStr := "" + if p.Limit > 0 { + paginationStr += ", " + fmt.Sprintf("first: %d", p.Limit) + } + if p.Offset > 0 { + paginationStr += ", " + fmt.Sprintf("offset: %d", p.Offset) + } else if p.After != "" { + paginationStr += ", " + fmt.Sprintf("after: %s", p.After) + } + if paginationStr == "" { + return "" + } + return paginationStr +} + +func sortingToQueryString(typeName string, s Sorting) string { + if s.OrderAscField == "" && s.OrderDescField == "" { + return "" + } + + var parts []string + first, second := s.OrderDescField, s.OrderAscField + firstOp, secondOp := "orderdesc", "orderasc" + + if !s.OrderDescFirst { + first, second = s.OrderAscField, s.OrderDescField + firstOp, secondOp = "orderasc", "orderdesc" + } + + if first != "" { + parts = append(parts, fmt.Sprintf("%s: %s", firstOp, getPredicateName(typeName, first))) + } + if second != "" { + parts = append(parts, fmt.Sprintf("%s: %s", secondOp, getPredicateName(typeName, second))) + } + + return ", " + strings.Join(parts, ", ") +} diff --git a/api_mutate_helper.go b/api_mutate_helper.go index 7957e0a..3d381c1 100644 --- a/api_mutate_helper.go +++ b/api_mutate_helper.go @@ -90,15 +90,10 @@ func generateCreateDqlMutationsAndSchema[T any](ctx context.Context, n *Namespac } if jsonToDbTags[jsonName] != nil { constraint := jsonToDbTags[jsonName].constraint - if constraint == "unique" || constraint == "term" { - uniqueConstraintFound = true - u.Directive = pb.SchemaUpdate_INDEX - if constraint == "unique" { - u.Tokenizer = []string{"exact"} - } else { - u.Tokenizer = []string{"term"} - } + if constraint == "vector" && valType != pb.Posting_VFLOAT { + return fmt.Errorf("vector index can only be applied to []float values") } + uniqueConstraintFound = addIndex(u, constraint, uniqueConstraintFound) } sch.Preds = append(sch.Preds, u) @@ -238,3 +233,39 @@ func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) return gid, nil } + +func addIndex(u *pb.SchemaUpdate, index string, uniqueConstraintExists bool) bool { + u.Directive = pb.SchemaUpdate_INDEX + switch index { + case "exact": + u.Tokenizer = []string{"exact"} + case "term": + u.Tokenizer = []string{"term"} + case "hash": + u.Tokenizer = []string{"hash"} + case "unique": + u.Tokenizer = []string{"exact"} + u.Unique = true + u.Upsert = true + uniqueConstraintExists = true + case "fulltext": + u.Tokenizer = []string{"fulltext"} + case "trigram": + u.Tokenizer = []string{"trigram"} + case "vector": + u.IndexSpecs = []*pb.VectorIndexSpec{ + { + Name: "hnsw", + Options: []*pb.OptionPair{ + { + Key: "metric", + Value: "cosine", + }, + }, + }, + } + default: + return uniqueConstraintExists + } + return uniqueConstraintExists +} diff --git a/api_query_helper.go b/api_query_helper.go index e62d8f3..a277d79 100644 --- a/api_query_helper.go +++ b/api_query_helper.go @@ -7,41 +7,40 @@ import ( "reflect" ) -func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, *T, error) { +func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, T, error) { return executeGet[T](ctx, n, gid) } -func getByGidWithObject[T any](ctx context.Context, n *Namespace, gid uint64, obj T) (uint64, *T, error) { +func getByGidWithObject[T any](ctx context.Context, n *Namespace, gid uint64, obj T) (uint64, T, error) { return executeGetWithObject[T](ctx, n, obj, false, gid) } -func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, *T, error) { +func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, T, error) { return executeGet[T](ctx, n, cf) } func getByConstrainedFieldWithObject[T any](ctx context.Context, n *Namespace, - cf ConstrainedField, obj T) (uint64, *T, error) { + cf ConstrainedField, obj T) (uint64, T, error) { return executeGetWithObject[T](ctx, n, obj, false, cf) } -func executeGet[T any, R UniqueField](ctx context.Context, n *Namespace, args ...R) (uint64, *T, error) { +func executeGet[T any, R UniqueField](ctx context.Context, n *Namespace, args ...R) (uint64, T, error) { + var obj T if len(args) != 1 { - return 0, nil, fmt.Errorf("expected 1 argument, got %d", len(args)) + return 0, obj, fmt.Errorf("expected 1 argument, got %d", len(args)) } - var obj T - return executeGetWithObject(ctx, n, obj, true, args...) } func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespace, - obj T, withReverse bool, args ...R) (uint64, *T, error) { + obj T, withReverse bool, args ...R) (uint64, T, error) { t := reflect.TypeOf(obj) fieldToJsonTags, jsonToDbTag, jsonToReverseEdgeTags, err := getFieldTags(t) if err != nil { - return 0, nil, err + return 0, obj, err } readFromQuery := "" if withReverse { @@ -64,16 +63,16 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac } else if cf, ok = any(args[0]).(ConstrainedField); ok { query = formatObjQuery(buildEqQuery(getPredicateName(t.Name(), cf.Key), cf.Value), readFromQuery) } else { - return 0, nil, fmt.Errorf("invalid unique field type") + return 0, obj, fmt.Errorf("invalid unique field type") } if jsonToDbTag[cf.Key] != nil && jsonToDbTag[cf.Key].constraint == "" { - return 0, nil, fmt.Errorf("constraint not defined for field %s", cf.Key) + return 0, obj, fmt.Errorf("constraint not defined for field %s", cf.Key) } resp, err := n.queryWithLock(ctx, query) if err != nil { - return 0, nil, err + return 0, obj, err } dynamicType := createDynamicStruct(t, fieldToJsonTags, 1) @@ -88,36 +87,122 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac // Unmarshal the JSON response into the dynamic struct if err := json.Unmarshal(resp.Json, &result); err != nil { - return 0, nil, err + return 0, obj, err } // Check if we have at least one object in the response if len(result.Obj) == 0 { - return 0, nil, ErrNoObjFound + return 0, obj, ErrNoObjFound } // Map the dynamic struct to the final type T finalObject := reflect.New(t).Interface() gid, err = mapDynamicToFinal(result.Obj[0], finalObject) if err != nil { - return 0, nil, err + return 0, obj, err + } + + if typedPtr, ok := finalObject.(*T); ok { + return gid, *typedPtr, nil + } + + if dirType, ok := finalObject.(T); ok { + return gid, dirType, nil } - // Convert to *interface{} then to *T - if ifacePtr, ok := finalObject.(*interface{}); ok { - if typedPtr, ok := (*ifacePtr).(*T); ok { - return gid, typedPtr, nil + return 0, obj, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) +} + +func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryParams, + withReverse bool) ([]uint64, []T, error) { + var obj T + t := reflect.TypeOf(obj) + fieldToJsonTags, _, jsonToReverseEdgeTags, err := getFieldTags(t) + if err != nil { + return nil, nil, err + } + + var filterQueryFunc QueryFunc = func() string { + return "" + } + var paginationAndSorting string + if queryParams.Filter != nil { + filterQueryFunc = filtersToQueryFunc(t.Name(), *queryParams.Filter) + } + if queryParams.Pagination != nil || queryParams.Sorting != nil { + var pagination, sorting string + if queryParams.Pagination != nil { + pagination = paginationToQueryString(*queryParams.Pagination) } + if queryParams.Sorting != nil { + sorting = sortingToQueryString(t.Name(), *queryParams.Sorting) + } + paginationAndSorting = fmt.Sprintf("%s %s", pagination, sorting) } - // If conversion fails, try direct conversion - if typedPtr, ok := finalObject.(*T); ok { - return gid, typedPtr, nil + readFromQuery := "" + if withReverse { + for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { + readFromQuery += fmt.Sprintf(` + %s: ~%s { + uid + expand(_all_) + dgraph.type + } + `, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) + } } - if dirType, ok := finalObject.(T); ok { - return gid, &dirType, nil + query := formatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) + + resp, err := n.queryWithLock(ctx, query) + if err != nil { + return nil, nil, err + } + + dynamicType := createDynamicStruct(t, fieldToJsonTags, 1) + + var result struct { + Objs []any `json:"objs"` + } + + var tempMap map[string][]any + if err := json.Unmarshal(resp.Json, &tempMap); err != nil { + return nil, nil, err + } + + // Determine the number of elements + numElements := len(tempMap["objs"]) + + // Append the interface the correct number of times + for i := 0; i < numElements; i++ { + result.Objs = append(result.Objs, reflect.New(dynamicType).Interface()) + } + + // Unmarshal the JSON response into the dynamic struct + if err := json.Unmarshal(resp.Json, &result); err != nil { + return nil, nil, err + } + + var gids []uint64 + var objs []T + for _, obj := range result.Objs { + finalObject := reflect.New(t).Interface() + gid, err := mapDynamicToFinal(obj, finalObject) + if err != nil { + return nil, nil, err + } + + if typedPtr, ok := finalObject.(*T); ok { + gids = append(gids, gid) + objs = append(objs, *typedPtr) + } else if dirType, ok := finalObject.(T); ok { + gids = append(gids, gid) + objs = append(objs, dirType) + } else { + return nil, nil, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) + } } - return 0, nil, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) + return gids, objs, nil } diff --git a/api_reflect.go b/api_reflect.go index f74cd53..832358e 100644 --- a/api_reflect.go +++ b/api_reflect.go @@ -183,7 +183,7 @@ func getUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { return gid, nil, nil } } - if jsonToDbTags[jsonName] != nil && jsonToDbTags[jsonName].constraint == "unique" { + if jsonToDbTags[jsonName] != nil && isValidUniqueIndex(jsonToDbTags[jsonName].constraint) { // check if value is zero or nil if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { continue @@ -197,3 +197,7 @@ func getUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { return 0, nil, fmt.Errorf(NoUniqueConstr, t.Name()) } + +func isValidUniqueIndex(name string) bool { + return name == "unique" +} diff --git a/api_test.go b/api_test.go index 1d3293c..701b30c 100644 --- a/api_test.go +++ b/api_test.go @@ -22,7 +22,7 @@ func TestFirstTimeUser(t *testing.T) { require.NoError(t, err) defer db.Close() - gid, user, err := modusdb.Create(db, &User{ + gid, user, err := modusdb.Create(db, User{ Name: "A", Age: 10, ClerkId: "123", @@ -59,7 +59,7 @@ func TestFirstTimeUser(t *testing.T) { _, queriedUser3, err := modusdb.Get[User](db, gid) require.Error(t, err) require.Equal(t, "no object found", err.Error()) - require.Nil(t, queriedUser3) + require.Equal(t, queriedUser3, User{}) } @@ -74,7 +74,7 @@ func TestCreateApi(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - user := &User{ + user := User{ Name: "B", Age: 20, ClerkId: "123", @@ -131,7 +131,7 @@ func TestCreateApiWithNonStruct(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - user := &User{ + user := User{ Name: "B", Age: 20, } @@ -152,7 +152,7 @@ func TestGetApi(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - user := &User{ + user := User{ Name: "B", Age: 20, ClerkId: "123", @@ -181,7 +181,7 @@ func TestGetApiWithConstrainedField(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - user := &User{ + user := User{ Name: "B", Age: 20, ClerkId: "123", @@ -213,7 +213,7 @@ func TestDeleteApi(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - user := &User{ + user := User{ Name: "B", Age: 20, ClerkId: "123", @@ -228,7 +228,7 @@ func TestDeleteApi(t *testing.T) { _, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) require.Error(t, err) require.Equal(t, "no object found", err.Error()) - require.Nil(t, queriedUser) + require.Equal(t, queriedUser, User{}) _, queriedUser, err = modusdb.Get[User](db, modusdb.ConstrainedField{ Key: "clerk_id", @@ -236,7 +236,7 @@ func TestDeleteApi(t *testing.T) { }, db1.ID()) require.Error(t, err) require.Equal(t, "no object found", err.Error()) - require.Nil(t, queriedUser) + require.Equal(t, queriedUser, User{}) } func TestUpsertApi(t *testing.T) { @@ -250,7 +250,7 @@ func TestUpsertApi(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - user := &User{ + user := User{ Name: "B", Age: 20, ClerkId: "123", @@ -273,6 +273,123 @@ func TestUpsertApi(t *testing.T) { require.Equal(t, "123", queriedUser.ClerkId) } +func TestQueryApi(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + users := []User{ + {Name: "A", Age: 10, ClerkId: "123"}, + {Name: "B", Age: 20, ClerkId: "123"}, + {Name: "C", Age: 30, ClerkId: "123"}, + {Name: "D", Age: 40, ClerkId: "123"}, + {Name: "E", Age: 50, ClerkId: "123"}, + } + + for _, user := range users { + _, _, err = modusdb.Create(db, user, db1.ID()) + require.NoError(t, err) + } + + gids, queriedUsers, err := modusdb.Query[User](db, modusdb.QueryParams{}, db1.ID()) + require.NoError(t, err) + require.Len(t, queriedUsers, 5) + require.Len(t, gids, 5) + require.Equal(t, "A", queriedUsers[0].Name) + require.Equal(t, "B", queriedUsers[1].Name) + require.Equal(t, "C", queriedUsers[2].Name) + require.Equal(t, "D", queriedUsers[3].Name) + require.Equal(t, "E", queriedUsers[4].Name) + + gids, queriedUsers, err = modusdb.Query[User](db, modusdb.QueryParams{ + Filter: &modusdb.Filter{ + Field: "age", + String: modusdb.StringPredicate{ + // The reason its a string even for int is bc i cant tell if + // user wants to compare with 0 the number or didn't provide a value + // TODO: fix this + GreaterOrEqual: fmt.Sprintf("%d", 20), + }, + }, + }, db1.ID()) + + require.NoError(t, err) + require.Len(t, queriedUsers, 4) + require.Len(t, gids, 4) + require.Equal(t, "B", queriedUsers[0].Name) + require.Equal(t, "C", queriedUsers[1].Name) + require.Equal(t, "D", queriedUsers[2].Name) + require.Equal(t, "E", queriedUsers[3].Name) +} + +func TestQueryApiWithPaginiationAndSorting(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + users := []User{ + {Name: "A", Age: 10, ClerkId: "123"}, + {Name: "B", Age: 20, ClerkId: "123"}, + {Name: "C", Age: 30, ClerkId: "123"}, + {Name: "D", Age: 40, ClerkId: "123"}, + {Name: "E", Age: 50, ClerkId: "123"}, + } + + for _, user := range users { + _, _, err = modusdb.Create(db, user, db1.ID()) + require.NoError(t, err) + } + + gids, queriedUsers, err := modusdb.Query[User](db, modusdb.QueryParams{ + Filter: &modusdb.Filter{ + Field: "age", + String: modusdb.StringPredicate{ + GreaterOrEqual: fmt.Sprintf("%d", 20), + }, + }, + Pagination: &modusdb.Pagination{ + Limit: 3, + Offset: 1, + }, + }, db1.ID()) + + require.NoError(t, err) + require.Len(t, queriedUsers, 3) + require.Len(t, gids, 3) + require.Equal(t, "C", queriedUsers[0].Name) + require.Equal(t, "D", queriedUsers[1].Name) + require.Equal(t, "E", queriedUsers[2].Name) + + gids, queriedUsers, err = modusdb.Query[User](db, modusdb.QueryParams{ + Pagination: &modusdb.Pagination{ + Limit: 3, + Offset: 1, + }, + Sorting: &modusdb.Sorting{ + OrderAscField: "age", + }, + }, db1.ID()) + + require.NoError(t, err) + require.Len(t, queriedUsers, 3) + require.Len(t, gids, 3) + require.Equal(t, "B", queriedUsers[0].Name) + require.Equal(t, "C", queriedUsers[1].Name) + require.Equal(t, "D", queriedUsers[2].Name) +} + type Project struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` @@ -298,7 +415,7 @@ func TestNestedObjectMutation(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - branch := &Branch{ + branch := Branch{ Name: "B", ClerkId: "123", Proj: Project{ @@ -352,7 +469,7 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - projGid, project, err := modusdb.Create(db, &Project{ + projGid, project, err := modusdb.Create(db, Project{ Name: "P", ClerkId: "456", }, db1.ID()) @@ -361,7 +478,7 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { require.Equal(t, "P", project.Name) require.Equal(t, project.Gid, projGid) - branch := &Branch{ + branch := Branch{ Name: "B", ClerkId: "123", Proj: Project{ @@ -415,7 +532,7 @@ func TestLinkingObjectsByGid(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - projGid, project, err := modusdb.Create(db, &Project{ + projGid, project, err := modusdb.Create(db, Project{ Name: "P", ClerkId: "456", }, db1.ID()) @@ -424,7 +541,7 @@ func TestLinkingObjectsByGid(t *testing.T) { require.Equal(t, "P", project.Name) require.Equal(t, project.Gid, projGid) - branch := &Branch{ + branch := Branch{ Name: "B", ClerkId: "123", Proj: Project{ @@ -489,7 +606,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { require.NoError(t, db1.DropData(ctx)) - branch := &BadBranch{ + branch := BadBranch{ Name: "B", ClerkId: "123", Proj: BadProject{ @@ -502,7 +619,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { require.Error(t, err) require.Equal(t, fmt.Sprintf(modusdb.NoUniqueConstr, "BadProject"), err.Error()) - proj := &BadProject{ + proj := BadProject{ Name: "P", ClerkId: "456", } @@ -512,3 +629,120 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { require.Equal(t, fmt.Sprintf(modusdb.NoUniqueConstr, "BadProject"), err.Error()) } + +type Document struct { + Gid uint64 `json:"gid,omitempty"` + Text string `json:"text,omitempty"` + TextVec []float32 `json:"textVec,omitempty" db:"constraint=vector"` +} + +func TestVectorIndexSearchTyped(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + documents := []Document{ + {Text: "apple", TextVec: []float32{0.1, 0.1, 0.0}}, + {Text: "banana", TextVec: []float32{0.0, 1.0, 0.0}}, + {Text: "carrot", TextVec: []float32{0.0, 0.0, 1.0}}, + {Text: "dog", TextVec: []float32{1.0, 1.0, 0.0}}, + {Text: "elephant", TextVec: []float32{0.0, 1.0, 1.0}}, + {Text: "fox", TextVec: []float32{1.0, 0.0, 1.0}}, + {Text: "gorilla", TextVec: []float32{1.0, 1.0, 1.0}}, + } + + for _, doc := range documents { + _, _, err = modusdb.Create(db, doc, db1.ID()) + require.NoError(t, err) + } + + const query = ` + { + documents(func: similar_to(Document.textVec, 5, "[0.1,0.1,0.1]")) { + Document.text + } + }` + + resp, err := db1.Query(ctx, query) + require.NoError(t, err) + require.JSONEq(t, `{ + "documents":[ + {"Document.text":"apple"}, + {"Document.text":"dog"}, + {"Document.text":"elephant"}, + {"Document.text":"fox"}, + {"Document.text":"gorilla"} + ] + }`, string(resp.GetJson())) + + const query2 = ` + { + documents(func: type("Document")) @filter(similar_to(Document.textVec, 5, "[0.1,0.1,0.1]")) { + Document.text + } + }` + + resp, err = db1.Query(ctx, query2) + require.NoError(t, err) + require.JSONEq(t, `{ + "documents":[ + {"Document.text":"apple"}, + {"Document.text":"dog"}, + {"Document.text":"elephant"}, + {"Document.text":"fox"}, + {"Document.text":"gorilla"} + ] + }`, string(resp.GetJson())) +} + +func TestVectorIndexSearchWithQuery(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + documents := []Document{ + {Text: "apple", TextVec: []float32{0.1, 0.1, 0.0}}, + {Text: "banana", TextVec: []float32{0.0, 1.0, 0.0}}, + {Text: "carrot", TextVec: []float32{0.0, 0.0, 1.0}}, + {Text: "dog", TextVec: []float32{1.0, 1.0, 0.0}}, + {Text: "elephant", TextVec: []float32{0.0, 1.0, 1.0}}, + {Text: "fox", TextVec: []float32{1.0, 0.0, 1.0}}, + {Text: "gorilla", TextVec: []float32{1.0, 1.0, 1.0}}, + } + + for _, doc := range documents { + _, _, err = modusdb.Create(db, doc, db1.ID()) + require.NoError(t, err) + } + + gids, docs, err := modusdb.Query[Document](db, modusdb.QueryParams{ + Filter: &modusdb.Filter{ + Field: "textVec", + Vector: modusdb.VectorPredicate{ + SimilarTo: []float32{0.1, 0.1, 0.1}, + TopK: 5, + }, + }, + }, db1.ID()) + + require.NoError(t, err) + require.Len(t, docs, 5) + require.Len(t, gids, 5) + require.Equal(t, "apple", docs[0].Text) + require.Equal(t, "dog", docs[1].Text) + require.Equal(t, "elephant", docs[2].Text) + require.Equal(t, "fox", docs[3].Text) + require.Equal(t, "gorilla", docs[4].Text) +} diff --git a/api_types.go b/api_types.go index 860edda..3e53bf8 100644 --- a/api_types.go +++ b/api_types.go @@ -4,10 +4,12 @@ import ( "context" "encoding/binary" "fmt" + "strings" "time" "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/types" "github.com/dgraph-io/dgraph/v24/x" "github.com/twpayne/go-geom" "github.com/twpayne/go-geom/encoding/wkb" @@ -26,6 +28,51 @@ type ConstrainedField struct { Value any } +type QueryParams struct { + Filter *Filter + Pagination *Pagination + Sorting *Sorting +} + +type Filter struct { + Field string + String StringPredicate + Vector VectorPredicate + And *Filter + Or *Filter + Not *Filter +} + +type Pagination struct { + Limit int64 + Offset int64 + After string +} + +type Sorting struct { + OrderAscField string + OrderDescField string + OrderDescFirst bool +} + +type StringPredicate struct { + Equals string + LessThan string + LessOrEqual string + GreaterThan string + GreaterOrEqual string + AllOfTerms []string + AnyOfTerms []string + AllOfText []string + AnyOfText []string + RegExp string +} + +type VectorPredicate struct { + SimilarTo []float32 + TopK int64 +} + type ModusDbOption func(*modusDbOptions) type modusDbOptions struct { @@ -110,6 +157,16 @@ func valueToApiVal(v any) (*api.Value, error) { return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil case float64: return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil + case []float32: + return &api.Value{Val: &api.Value_Vfloat32Val{ + Vfloat32Val: types.FloatArrayAsBytes(val)}}, nil + case []float64: + float32Slice := make([]float32, len(val)) + for i, v := range val { + float32Slice[i] = float32(v) + } + return &api.Value{Val: &api.Value_Vfloat32Val{ + Vfloat32Val: types.FloatArrayAsBytes(float32Slice)}}, nil case []byte: return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil case time.Time: @@ -130,3 +187,54 @@ func valueToApiVal(v any) (*api.Value, error) { return nil, fmt.Errorf("unsupported type %T", v) } } + +func filterToQueryFunc(typeName string, f Filter) QueryFunc { + // Handle logical operators first + if f.And != nil { + return And(filterToQueryFunc(typeName, *f.And)) + } + if f.Or != nil { + return Or(filterToQueryFunc(typeName, *f.Or)) + } + if f.Not != nil { + return Not(filterToQueryFunc(typeName, *f.Not)) + } + + // Handle field predicates + if f.String.Equals != "" { + return buildEqQuery(getPredicateName(typeName, f.Field), f.String.Equals) + } + if len(f.String.AllOfTerms) != 0 { + return buildAllOfTermsQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AllOfTerms, " ")) + } + if len(f.String.AnyOfTerms) != 0 { + return buildAnyOfTermsQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AnyOfTerms, " ")) + } + if len(f.String.AllOfText) != 0 { + return buildAllOfTextQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AllOfText, " ")) + } + if len(f.String.AnyOfText) != 0 { + return buildAnyOfTextQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AnyOfText, " ")) + } + if f.String.RegExp != "" { + return buildRegExpQuery(getPredicateName(typeName, f.Field), f.String.RegExp) + } + if f.String.LessThan != "" { + return buildLtQuery(getPredicateName(typeName, f.Field), f.String.LessThan) + } + if f.String.LessOrEqual != "" { + return buildLeQuery(getPredicateName(typeName, f.Field), f.String.LessOrEqual) + } + if f.String.GreaterThan != "" { + return buildGtQuery(getPredicateName(typeName, f.Field), f.String.GreaterThan) + } + if f.String.GreaterOrEqual != "" { + return buildGeQuery(getPredicateName(typeName, f.Field), f.String.GreaterOrEqual) + } + if f.Vector.SimilarTo != nil { + return buildSimilarToQuery(getPredicateName(typeName, f.Field), f.Vector.TopK, f.Vector.SimilarTo) + } + + // Return empty query if no conditions match + return func() string { return "" } +} From f7007a48f23e0ea085f81e34ce8df8f59b36bf89 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Thu, 2 Jan 2025 13:37:32 -0500 Subject: [PATCH 11/23] chore: repo setup (#43) This PR adds the initial issue and PR templates, a changelog, code of conduct, contributing guide, and security reporting instructions. We can enhance these over time. --- .github/ISSUE_TEMPLATE/bug_report.md | 34 ++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++ .github/pull_request_template.md | 20 ++++ CHANGELOG.md | 7 ++ CODE_OF_CONDUCT.md | 128 ++++++++++++++++++++++ CONTRIBUTING.md | 56 ++++++++++ SECURITY.md | 7 ++ 8 files changed, 277 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..446d7ee --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Environment** + +- OS: [e.g. macOS, Windows, Ubuntu] +- Language [e.g. Go] +- Version [e.g. v0.xx] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..8b81b4b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Modus Community Support + url: https://discord.hypermode.com + about: Please ask and answer questions here diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..d70ba26 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,20 @@ +**Description** + +Please explain the changes you made here. + +**Checklist** + +- [ ] Code compiles correctly and linting passes locally +- [ ] For all _code_ changes, an entry added to the `CHANGELOG.md` file describing and linking to this PR +- [ ] Tests added for new functionality, or regression tests for bug fixes added as applicable +- [ ] For public APIs, new features, etc., PR on [docs repo](https://github.com/hypermodeinc/docs) staged and linked here + +**Instructions** + +- The PR title should follow the [Conventional Commits](https://www.conventionalcommits.org/) syntax, leading with `fix:`, `feat:`, `chore:`, `ci:`, etc. +- The description should briefly explain what the PR is about. In the case of a bugfix, describe or link to the bug. +- In the checklist section, check the boxes in that are applicable, using `[x]` syntax. + - If not applicable, remove the entire line. Only leave the box unchecked if you intend to come back and check the box later. +- Delete the `Instructions` line and everything below it, to indicate you have read and are following these instructions. 🙂 + +Thank you for your contribution to modusDB! diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2999085 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## 2025-01-02 - Version 0.1.0 + +Baseline for the changelog. + +See git commit history for changes for this version and prior. diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..1f59696 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +hello@hypermode.com. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4aca915 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing to modusDB + +We're really glad you're here and would love for you to contribute to modusDB! There are a variety of ways to make modusDB better, including bug fixes, features, docs, and blog posts (among others). Every bit helps 🙏 + +Please help us keep the community safe while working on the project by upholding our [Code of Conduct](/CODE_OF_CONDUCT.md) at all times. + +Before jumping to a pull request, ensure you've looked at [PRs](https://github.com/hypermodeinc/modusdb/pulls) and [issues](https://github.com/hypermodeinc/modusdb/issues) (open and closed) for existing work related to your idea. + +If in doubt or contemplating a larger change, join the [Hypermode Discord](https://discord.hypermode.com) and start a discussion in the [#modus](https://discord.com/channels/1267579648657850441/1292948253796466730) channel. + +## Codebase + +The development language of modusDB is Go. + +### Development environment + +The fastest path to setting up a development environment for modusDB is through VS Code. The repo includes a set of configs to set VS Code up automatically. + +### Clone the Modus repository + +To contribute code, start by forking the Modus repository. In the top-right of the [repo](https://github.com/hypermodeinc/modusdb), click **Fork**. Follow the instructions to create a fork of the repo in your GitHub workspace. + +### Building and running tests + +Wherever possible, we use the built-in language capabilities. For example, unit tests can be run with: + +```bash +go test ./... +``` + +### Opening a pull request + +When you're ready, open a pull request against the `main` branch in the modusDB repo. Include a clear, detailed description of the changes you've made. Be sure to add and update tests and docs as needed. + +We do our best to respond to PRs within a few days. If you've not heard back, feel free to ping on Discord. + +## Other ways to help + +Pull requests are awesome, but there are many ways to help. + +### Documentation improvements + +Modus docs are maintained in a [separate repository](https://github.com/hypermodeinc/docs). Relevant updates and issues should be opened in that repo. + +### Blogging and presenting your work + +Share what you're building with Modus in your preferred medium. We'd love to help amplify your work and/or provide feedback, so get in touch if you'd like some help! + +### Join the community + +There are lots of people building with modusDB who are excited to connect! + +- Chat on [Discord](https://discord.hypermode.com) +- Join the conversation on [X](https://x.com/hypermodeinc) +- Read the latest posts on the [Blog](https://hypermode.com/blog) +- Connect with us on [LinkedIn](https://linkedin.com/company/hypermode) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e698ff6 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Reporting Security Concerns + +We take the security of Modus very seriously. If you believe you have found a security vulnerability in Modus, we encourage you to let us know right away. + +We will investigate all legitimate reports and do our best to quickly fix the problem. Please report any issues or vulnerabilities via Github Security Advisories instead of posting a public issue in GitHub. You can also send security communications to security@hypermode.com. + +Please include the version identifier and details on how the vulnerability can be exploited. From fff8aaaa9e8f5dd8ca37fd7839f155e72c273b21 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Thu, 2 Jan 2025 13:37:40 -0500 Subject: [PATCH 12/23] chore: create LICENSE (#42) --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From e676bc656ada9e403668677f92424b0525e2b180 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Thu, 2 Jan 2025 16:10:23 -0500 Subject: [PATCH 13/23] chore: add trunk (#44) **Description** Initialize Trunk for broader linting and security monitoring. Fix addressable issues. **Checklist** - [x] Code compiles correctly and linting passes locally --- .github/renovate.json | 4 +- .github/workflows/ci-go-lint.yaml | 6 +-- .github/workflows/ci-go-tests.yaml | 6 +-- .trunk/.gitignore | 9 ++++ .../configs/.golangci.yaml | 0 .trunk/configs/.markdownlint.yaml | 2 + .trunk/configs/.yamllint.yaml | 7 +++ .trunk/trunk.yaml | 43 +++++++++++++++++++ .vscode/extensions.json | 3 ++ .vscode/settings.json | 6 +++ README.md | 3 +- 11 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 .trunk/.gitignore rename .golangci.yaml => .trunk/configs/.golangci.yaml (100%) create mode 100644 .trunk/configs/.markdownlint.yaml create mode 100644 .trunk/configs/.yamllint.yaml create mode 100644 .trunk/trunk.yaml create mode 100644 .vscode/extensions.json create mode 100644 .vscode/settings.json diff --git a/.github/renovate.json b/.github/renovate.json index 77df0c5..6bd88fb 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,6 +1,4 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "github>hypermodeinc/renovate-config" - ] + "extends": ["github>hypermodeinc/renovate-config"] } diff --git a/.github/workflows/ci-go-lint.yaml b/.github/workflows/ci-go-lint.yaml index e5bcfd8..14cb093 100644 --- a/.github/workflows/ci-go-lint.yaml +++ b/.github/workflows/ci-go-lint.yaml @@ -8,8 +8,8 @@ on: - reopened - ready_for_review paths: - - '**/*.go' - - '**/go.mod' + - "**/*.go" + - "**/go.mod" permissions: contents: read @@ -25,7 +25,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version-file: go.mod cache-dependency-path: go.sum - name: golangci-lint diff --git a/.github/workflows/ci-go-tests.yaml b/.github/workflows/ci-go-tests.yaml index 579348c..a0dc02d 100644 --- a/.github/workflows/ci-go-tests.yaml +++ b/.github/workflows/ci-go-tests.yaml @@ -8,8 +8,8 @@ on: - reopened - ready_for_review paths: - - '**/*.go' - - '**/go.mod' + - "**/*.go" + - "**/go.mod" permissions: contents: read @@ -25,7 +25,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version-file: go.mod cache-dependency-path: go.sum - name: Run Unit Tests diff --git a/.trunk/.gitignore b/.trunk/.gitignore new file mode 100644 index 0000000..15966d0 --- /dev/null +++ b/.trunk/.gitignore @@ -0,0 +1,9 @@ +*out +*logs +*actions +*notifications +*tools +plugins +user_trunk.yaml +user.yaml +tmp diff --git a/.golangci.yaml b/.trunk/configs/.golangci.yaml similarity index 100% rename from .golangci.yaml rename to .trunk/configs/.golangci.yaml diff --git a/.trunk/configs/.markdownlint.yaml b/.trunk/configs/.markdownlint.yaml new file mode 100644 index 0000000..b40ee9d --- /dev/null +++ b/.trunk/configs/.markdownlint.yaml @@ -0,0 +1,2 @@ +# Prettier friendly markdownlint config (all formatting rules disabled) +extends: markdownlint/style/prettier diff --git a/.trunk/configs/.yamllint.yaml b/.trunk/configs/.yamllint.yaml new file mode 100644 index 0000000..184e251 --- /dev/null +++ b/.trunk/configs/.yamllint.yaml @@ -0,0 +1,7 @@ +rules: + quoted-strings: + required: only-when-needed + extra-allowed: ["{|}"] + key-duplicates: {} + octal-values: + forbid-implicit-octal: true diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml new file mode 100644 index 0000000..57bc1f8 --- /dev/null +++ b/.trunk/trunk.yaml @@ -0,0 +1,43 @@ +# This file controls the behavior of Trunk: https://docs.trunk.io/cli +# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml + +version: 0.1 + +cli: + version: 1.22.8 +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) + +plugins: + sources: + - id: trunk + ref: v1.6.6 + uri: https://github.com/trunk-io/plugins +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) + +runtimes: + enabled: + - go@1.23.3 + - node@18.20.5 + - python@3.10.8 +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) + +lint: + enabled: + - actionlint@1.7.5 + - checkov@3.2.346 + - git-diff-check + - gofmt@1.20.4 + - golangci-lint@1.62.2 + - markdownlint@0.43.0 + - osv-scanner@1.9.2 + - prettier@3.4.2 + - renovate@39.88.0 + - trufflehog@3.88.0 + - yamllint@1.35.1 + +actions: + enabled: + - trunk-announce + - trunk-check-pre-push + - trunk-fmt-pre-commit + - trunk-upgrade-available diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..29d4338 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,3 @@ +{ + "recommendations": ["trunk.io"] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..93ff3ac --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "editor.formatOnSave": true, + "editor.defaultFormatter": "trunk.io", + "editor.trimAutoWhitespace": true, + "trunk.autoInit": false +} diff --git a/README.md b/README.md index 2283ef2..5c4d16e 100644 --- a/README.md +++ b/README.md @@ -1,2 +1 @@ -ModusDB -====== +# ModusDB From a157525af89807c07c009f7914615e7248e19e3e Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:56:12 -0500 Subject: [PATCH 14/23] chore: add readme (#46) **Description** add readme to repo --- .github/ISSUE_TEMPLATE/bug_report.md | 20 ++-- .github/ISSUE_TEMPLATE/feature_request.md | 21 ++-- .github/pull_request_template.md | 20 +++- .trunk/configs/.markdownlint.json | 7 ++ .trunk/configs/.markdownlint.yaml | 2 - .trunk/configs/.prettierrc | 5 + CODE_OF_CONDUCT.md | 135 ++++++++++------------ CONTRIBUTING.md | 39 +++++-- README.md | 100 +++++++++++++++- SECURITY.md | 7 +- 10 files changed, 234 insertions(+), 122 deletions(-) create mode 100644 .trunk/configs/.markdownlint.json delete mode 100644 .trunk/configs/.markdownlint.yaml create mode 100644 .trunk/configs/.prettierrc diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 446d7ee..e3f9ccf 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,28 +1,23 @@ --- name: Bug report about: Create a report to help us improve -title: '' +title: "" labels: bug -assignees: '' - +assignees: "" --- -**Describe the bug** -A clear and concise description of what the bug is. +**Describe the bug** A clear and concise description of what the bug is. -**To Reproduce** -Steps to reproduce the behavior: +**To Reproduce** Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error -**Expected behavior** -A clear and concise description of what you expected to happen. +**Expected behavior** A clear and concise description of what you expected to happen. -**Screenshots** -If applicable, add screenshots to help explain your problem. +**Screenshots** If applicable, add screenshots to help explain your problem. **Environment** @@ -30,5 +25,4 @@ If applicable, add screenshots to help explain your problem. - Language [e.g. Go] - Version [e.g. v0.xx] -**Additional context** -Add any other context about the problem here. +**Additional context** Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7..3a3c141 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,20 +1,17 @@ --- name: Feature request about: Suggest an idea for this project -title: '' -labels: '' -assignees: '' - +title: "" +labels: "" +assignees: "" --- -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] +**Is your feature request related to a problem? Please describe.** A clear and concise description +of what the problem is. Ex. I'm always frustrated when [...] -**Describe the solution you'd like** -A clear and concise description of what you want to happen. +**Describe the solution you'd like** A clear and concise description of what you want to happen. -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. +**Describe alternatives you've considered** A clear and concise description of any alternative +solutions or features you've considered. -**Additional context** -Add any other context or screenshots about the feature request here. +**Additional context** Add any other context or screenshots about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d70ba26..e62dfbf 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,3 +1,5 @@ + + **Description** Please explain the changes you made here. @@ -5,16 +7,22 @@ Please explain the changes you made here. **Checklist** - [ ] Code compiles correctly and linting passes locally -- [ ] For all _code_ changes, an entry added to the `CHANGELOG.md` file describing and linking to this PR +- [ ] For all _code_ changes, an entry added to the `CHANGELOG.md` file describing and linking to + this PR - [ ] Tests added for new functionality, or regression tests for bug fixes added as applicable -- [ ] For public APIs, new features, etc., PR on [docs repo](https://github.com/hypermodeinc/docs) staged and linked here +- [ ] For public APIs, new features, etc., PR on [docs repo](https://github.com/hypermodeinc/docs) + staged and linked here **Instructions** -- The PR title should follow the [Conventional Commits](https://www.conventionalcommits.org/) syntax, leading with `fix:`, `feat:`, `chore:`, `ci:`, etc. -- The description should briefly explain what the PR is about. In the case of a bugfix, describe or link to the bug. +- The PR title should follow the [Conventional Commits](https://www.conventionalcommits.org/) + syntax, leading with `fix:`, `feat:`, `chore:`, `ci:`, etc. +- The description should briefly explain what the PR is about. In the case of a bugfix, describe or + link to the bug. - In the checklist section, check the boxes in that are applicable, using `[x]` syntax. - - If not applicable, remove the entire line. Only leave the box unchecked if you intend to come back and check the box later. -- Delete the `Instructions` line and everything below it, to indicate you have read and are following these instructions. 🙂 + - If not applicable, remove the entire line. Only leave the box unchecked if you intend to come + back and check the box later. +- Delete the `Instructions` line and everything below it, to indicate you have read and are + following these instructions. 🙂 Thank you for your contribution to modusDB! diff --git a/.trunk/configs/.markdownlint.json b/.trunk/configs/.markdownlint.json new file mode 100644 index 0000000..a7c5d5c --- /dev/null +++ b/.trunk/configs/.markdownlint.json @@ -0,0 +1,7 @@ +{ + "line-length": { "line_length": 150, "tables": false }, + "no-inline-html": false, + "no-bare-urls": false, + "no-space-in-emphasis": false, + "no-emphasis-as-heading": false +} diff --git a/.trunk/configs/.markdownlint.yaml b/.trunk/configs/.markdownlint.yaml deleted file mode 100644 index b40ee9d..0000000 --- a/.trunk/configs/.markdownlint.yaml +++ /dev/null @@ -1,2 +0,0 @@ -# Prettier friendly markdownlint config (all formatting rules disabled) -extends: markdownlint/style/prettier diff --git a/.trunk/configs/.prettierrc b/.trunk/configs/.prettierrc new file mode 100644 index 0000000..577642c --- /dev/null +++ b/.trunk/configs/.prettierrc @@ -0,0 +1,5 @@ +{ + "semi": false, + "proseWrap": "always", + "printWidth": 100 +} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 1f59696..614af54 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -2,124 +2,109 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our -community a harassment-free experience for everyone, regardless of age, body -size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. +We as members, contributors, and leaders pledge to make participation in our community a +harassment-free experience for everyone, regardless of age, body size, visible or invisible +disability, ethnicity, sex characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, religion, or sexual +identity and orientation. -We pledge to act and interact in ways that contribute to an open, welcoming, -diverse, inclusive, and healthy community. +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and +healthy community. ## Our Standards -Examples of behavior that contributes to a positive environment for our -community include: +Examples of behavior that contributes to a positive environment for our community include: -* Demonstrating empathy and kindness toward other people -* Being respectful of differing opinions, viewpoints, and experiences -* Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, - and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the + experience +- Focusing on what is best not just for us as individuals, but for the overall community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind -* Trolling, insulting or derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, without their + explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities -Community leaders are responsible for clarifying and enforcing our standards of -acceptable behavior and will take appropriate and fair corrective action in -response to any behavior that they deem inappropriate, threatening, offensive, -or harmful. +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior +and will take appropriate and fair corrective action in response to any behavior that they deem +inappropriate, threatening, offensive, or harmful. -Community leaders have the right and responsibility to remove, edit, or reject -comments, commits, code, wiki edits, issues, and other contributions that are -not aligned to this Code of Conduct, and will communicate reasons for moderation -decisions when appropriate. +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, +code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and +will communicate reasons for moderation decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when -an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, -posting via an official social media account, or acting as an appointed -representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when an individual is +officially representing the community in public spaces. Examples of representing our community +include using an official e-mail address, posting via an official social media account, or acting as +an appointed representative at an online or offline event. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -hello@hypermode.com. -All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community +leaders responsible for enforcement at hello@hypermode.com. All complaints will be reviewed and +investigated promptly and fairly. -All community leaders are obligated to respect the privacy and security of the -reporter of any incident. +All community leaders are obligated to respect the privacy and security of the reporter of any +incident. ## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining -the consequences for any action they deem in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining the consequences for +any action they deem in violation of this Code of Conduct: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior deemed -unprofessional or unwelcome in the community. +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or +unwelcome in the community. -**Consequence**: A private, written warning from community leaders, providing -clarity around the nature of the violation and an explanation of why the -behavior was inappropriate. A public apology may be requested. +**Consequence**: A private, written warning from community leaders, providing clarity around the +nature of the violation and an explanation of why the behavior was inappropriate. A public apology +may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: A violation through a single incident or series of actions. -**Consequence**: A warning with consequences for continued behavior. No -interaction with the people involved, including unsolicited interaction with -those enforcing the Code of Conduct, for a specified period of time. This -includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. +**Consequence**: A warning with consequences for continued behavior. No interaction with the people +involved, including unsolicited interaction with those enforcing the Code of Conduct, for a +specified period of time. This includes avoiding interactions in community spaces as well as +external channels like social media. Violating these terms may lead to a temporary or permanent ban. ### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, including -sustained inappropriate behavior. +**Community Impact**: A serious violation of community standards, including sustained inappropriate +behavior. -**Consequence**: A temporary ban from any sort of interaction or public -communication with the community for a specified period of time. No public or -private interaction with the people involved, including unsolicited interaction -with those enforcing the Code of Conduct, is allowed during this period. -Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public communication with the +community for a specified period of time. No public or private interaction with the people involved, +including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this +period. Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an -individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community standards, including +sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement +of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within -the community. +**Consequence**: A permanent ban from any sort of public interaction within the community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. -Community Impact Guidelines were inspired by [Mozilla's code of conduct -enforcement ladder](https://github.com/mozilla/diversity). +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4aca915..b119d3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,12 +1,20 @@ # Contributing to modusDB -We're really glad you're here and would love for you to contribute to modusDB! There are a variety of ways to make modusDB better, including bug fixes, features, docs, and blog posts (among others). Every bit helps 🙏 +We're really glad you're here and would love for you to contribute to modusDB! There are a variety +of ways to make modusDB better, including bug fixes, features, docs, and blog posts (among others). +Every bit helps 🙏 -Please help us keep the community safe while working on the project by upholding our [Code of Conduct](/CODE_OF_CONDUCT.md) at all times. +Please help us keep the community safe while working on the project by upholding our +[Code of Conduct](/CODE_OF_CONDUCT.md) at all times. -Before jumping to a pull request, ensure you've looked at [PRs](https://github.com/hypermodeinc/modusdb/pulls) and [issues](https://github.com/hypermodeinc/modusdb/issues) (open and closed) for existing work related to your idea. +Before jumping to a pull request, ensure you've looked at +[PRs](https://github.com/hypermodeinc/modusdb/pulls) and +[issues](https://github.com/hypermodeinc/modusdb/issues) (open and closed) for existing work related +to your idea. -If in doubt or contemplating a larger change, join the [Hypermode Discord](https://discord.hypermode.com) and start a discussion in the [#modus](https://discord.com/channels/1267579648657850441/1292948253796466730) channel. +If in doubt or contemplating a larger change, join the +[Hypermode Discord](https://discord.hypermode.com) and start a discussion in the +[#modus](https://discord.com/channels/1267579648657850441/1292948253796466730) channel. ## Codebase @@ -14,15 +22,19 @@ The development language of modusDB is Go. ### Development environment -The fastest path to setting up a development environment for modusDB is through VS Code. The repo includes a set of configs to set VS Code up automatically. +The fastest path to setting up a development environment for modusDB is through VS Code. The repo +includes a set of configs to set VS Code up automatically. ### Clone the Modus repository -To contribute code, start by forking the Modus repository. In the top-right of the [repo](https://github.com/hypermodeinc/modusdb), click **Fork**. Follow the instructions to create a fork of the repo in your GitHub workspace. +To contribute code, start by forking the Modus repository. In the top-right of the +[repo](https://github.com/hypermodeinc/modusdb), click **Fork**. Follow the instructions to create a +fork of the repo in your GitHub workspace. ### Building and running tests -Wherever possible, we use the built-in language capabilities. For example, unit tests can be run with: +Wherever possible, we use the built-in language capabilities. For example, unit tests can be run +with: ```bash go test ./... @@ -30,9 +42,12 @@ go test ./... ### Opening a pull request -When you're ready, open a pull request against the `main` branch in the modusDB repo. Include a clear, detailed description of the changes you've made. Be sure to add and update tests and docs as needed. +When you're ready, open a pull request against the `main` branch in the modusDB repo. Include a +clear, detailed description of the changes you've made. Be sure to add and update tests and docs as +needed. -We do our best to respond to PRs within a few days. If you've not heard back, feel free to ping on Discord. +We do our best to respond to PRs within a few days. If you've not heard back, feel free to ping on +Discord. ## Other ways to help @@ -40,11 +55,13 @@ Pull requests are awesome, but there are many ways to help. ### Documentation improvements -Modus docs are maintained in a [separate repository](https://github.com/hypermodeinc/docs). Relevant updates and issues should be opened in that repo. +Modus docs are maintained in a [separate repository](https://github.com/hypermodeinc/docs). Relevant +updates and issues should be opened in that repo. ### Blogging and presenting your work -Share what you're building with Modus in your preferred medium. We'd love to help amplify your work and/or provide feedback, so get in touch if you'd like some help! +Share what you're building with Modus in your preferred medium. We'd love to help amplify your work +and/or provide feedback, so get in touch if you'd like some help! ### Join the community diff --git a/README.md b/README.md index 5c4d16e..3c9493b 100644 --- a/README.md +++ b/README.md @@ -1 +1,99 @@ -# ModusDB + +
+ +[![modus](https://github.com/user-attachments/assets/1a6020bd-d041-4dd0-b4a9-ce01dc015b65)](https://github.com/hypermodeinc/modusdb) + +[![GitHub License](https://img.shields.io/github/license/hypermodeinc/modusdb)](https://github.com/hypermodeinc/modusdb?tab=Apache-2.0-1-ov-file#readme) +[![chat](https://img.shields.io/discord/1267579648657850441)](https://discord.gg/NJQ4bJpffF) +[![GitHub Repo stars](https://img.shields.io/github/stars/hypermodeinc/modusdb)](https://github.com/hypermodeinc/modusdb/stargazers) +[![GitHub commit activity](https://img.shields.io/github/commit-activity/m/hypermodeinc/modusdb)](https://github.com/hypermodeinc/modusdb/commits/main/) + +
+ +

+ Docs + · + Discord +

+ +**ModusDB is a high-performance, transactional database system.** It's designed to be type-first, +schema-agnostic, and portable. ModusDB provides object-oriented APIs that makes it simple to build +new apps, paired with support for advanced use cases through the Dgraph Query Language (DQL). A +dynamic schema allows for natural relations to be expressed in your data with performance that +scales with your use case. + +ModusDB is available as a Go package for running in-process, providing low-latency reads, writes, +and vector searches. We’ve made trade-offs to prioritize speed and simplicity. + +The [modus framework](https://github.com/hypermodeinc/modus) is optimized for apps that require +sub-second response times. ModusDB augments polyglot functions with simple to use data and vector +storage. When paired together, you can build a complete AI semantic search or retrieval-augmented +generation (RAG) feature with a single framework. + +## Quickstart + +```go +package main + +import ( + "github.com/hypermodeinc/modusdb" +) + +type User struct { + Gid uint64 `json:"gid,omitempty"` + Id string `json:"id,omitempty" db:"constraint=unique"` + Name string `json:"name,omitempty"` + Age int `json:"age,omitempty"` +} + +func main() { + db, err := New(NewDefaultConfig("/tmp/modusdb")) + if err != nil { + panic(err) + } + defer db.Close() + + gid, user, err := modusdb.Upsert(db, User{ + Id: "123", + Name: "A", + Age: 10, + }) + if err != nil { + panic(err) + } + fmt.Println(user) + + _, queriedUser, err := modusdb.Get[User](db, gid) + if err != nil { + panic(err) + } + fmt.Println(queriedUser) + + _, _, err = modusdb.Delete[User](db, gid) + if err != nil { + panic(err) + } +} +``` + +## Open Source + +The modus framework, including modusDB, is developed by [Hypermode](https://hypermode.com/) as an +open-source project, integral but independent from Hypermode. + +We welcome external contributions. See the [CONTRIBUTING.md](./CONTRIBUTING.md) file if you would +like to get involved. + +Modus and its components are Copyright 2025 Hypermode Inc., and licensed under the terms of the +Apache License, Version 2.0. See the [LICENSE](./LICENSE) file for a complete copy of the license. +If you have any questions about modus licensing, or need an alternate license or other arrangement, +please contact us at hello@hypermode.com. + +## Acknowledgements + +ModusDB builds heavily upon packages from the open source projects of +[Dgraph](https://github.com/dgraph-io/dgraph) (graph query processing and transaction management), +[Badger](https://github.com/dgraph-io/badger) (data storage), and +[Ristretto](https://github.com/dgraph-io/ristretto) (cache). We expect the architecture and +implementations of modusDB and Dgraph to expand in differentiation over time as the projects +optimize for different core use cases, while maintaining Dgraph Query Language (DQL) compatibility. diff --git a/SECURITY.md b/SECURITY.md index e698ff6..65d28b6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,7 +1,10 @@ # Reporting Security Concerns -We take the security of Modus very seriously. If you believe you have found a security vulnerability in Modus, we encourage you to let us know right away. +We take the security of Modus very seriously. If you believe you have found a security vulnerability +in Modus, we encourage you to let us know right away. -We will investigate all legitimate reports and do our best to quickly fix the problem. Please report any issues or vulnerabilities via Github Security Advisories instead of posting a public issue in GitHub. You can also send security communications to security@hypermode.com. +We will investigate all legitimate reports and do our best to quickly fix the problem. Please report +any issues or vulnerabilities via Github Security Advisories instead of posting a public issue in +GitHub. You can also send security communications to security@hypermode.com. Please include the version identifier and details on how the vulnerability can be exploited. From dac39a7f7360467ed7fd13398c5c0f376eb37c6f Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 2 Jan 2025 17:49:38 -0800 Subject: [PATCH 15/23] chore: add copyrights (#47) **Description** Adds copyrights to all go files **Checklist** - [x] Code compiles correctly and linting passes locally - [ ] For all _code_ changes, an entry added to the `CHANGELOG.md` file describing and linking to this PR - [x] Tests added for new functionality, or regression tests for bug fixes added as applicable - [ ] For public APIs, new features, etc., PR on [docs repo](https://github.com/hypermodeinc/docs) staged and linked here Thank you for your contribution to modusDB! --- api.go | 9 +++++++++ api_dql.go | 9 +++++++++ api_mutate_helper.go | 9 +++++++++ api_query_helper.go | 9 +++++++++ api_reflect.go | 9 +++++++++ api_test.go | 9 +++++++++ api_types.go | 9 +++++++++ config.go | 9 +++++++++ db.go | 9 +++++++++ db_test.go | 9 +++++++++ live.go | 9 +++++++++ live_benchmark_test.go | 9 +++++++++ live_test.go | 9 +++++++++ namespace.go | 9 +++++++++ namespace_test.go | 9 +++++++++ utils.go | 9 +++++++++ vector_test.go | 9 +++++++++ zero.go | 9 +++++++++ 18 files changed, 162 insertions(+) diff --git a/api.go b/api.go index e329198..379fccd 100644 --- a/api.go +++ b/api.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/api_dql.go b/api_dql.go index b20ecf0..203440a 100644 --- a/api_dql.go +++ b/api_dql.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/api_mutate_helper.go b/api_mutate_helper.go index 3d381c1..2dd4188 100644 --- a/api_mutate_helper.go +++ b/api_mutate_helper.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/api_query_helper.go b/api_query_helper.go index a277d79..5c2552f 100644 --- a/api_query_helper.go +++ b/api_query_helper.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/api_reflect.go b/api_reflect.go index 832358e..1779a81 100644 --- a/api_reflect.go +++ b/api_reflect.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/api_test.go b/api_test.go index 701b30c..0ae4bea 100644 --- a/api_test.go +++ b/api_test.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb_test import ( diff --git a/api_types.go b/api_types.go index 3e53bf8..2384f35 100644 --- a/api_types.go +++ b/api_types.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/config.go b/config.go index e33a0ad..76d3d1b 100644 --- a/config.go +++ b/config.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb type Config struct { diff --git a/db.go b/db.go index 0f32a56..d092ecd 100644 --- a/db.go +++ b/db.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/db_test.go b/db_test.go index bbcc9ae..5de5d68 100644 --- a/db_test.go +++ b/db_test.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb_test import ( diff --git a/live.go b/live.go index 3e89e05..d786fab 100644 --- a/live.go +++ b/live.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/live_benchmark_test.go b/live_benchmark_test.go index 5efd4b2..62aee86 100644 --- a/live_benchmark_test.go +++ b/live_benchmark_test.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb_test import ( diff --git a/live_test.go b/live_test.go index d6b4880..4b7cf88 100644 --- a/live_test.go +++ b/live_test.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb_test import ( diff --git a/namespace.go b/namespace.go index 6d9f1af..710a1e0 100644 --- a/namespace.go +++ b/namespace.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/namespace_test.go b/namespace_test.go index 655e216..3f808bc 100644 --- a/namespace_test.go +++ b/namespace_test.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb_test import ( diff --git a/utils.go b/utils.go index 133ee1d..e571bb2 100644 --- a/utils.go +++ b/utils.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( diff --git a/vector_test.go b/vector_test.go index 135d438..e62057d 100644 --- a/vector_test.go +++ b/vector_test.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb_test import ( diff --git a/zero.go b/zero.go index d790a70..130e030 100644 --- a/zero.go +++ b/zero.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package modusdb import ( From 77effe0e90b2265c2ab552038c6b4fdbbf1e2fce Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Thu, 2 Jan 2025 21:07:40 -0500 Subject: [PATCH 16/23] Update trunk.yaml --- .trunk/trunk.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 57bc1f8..85fae60 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -5,22 +5,22 @@ version: 0.1 cli: version: 1.22.8 -# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) +# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) plugins: sources: - id: trunk ref: v1.6.6 uri: https://github.com/trunk-io/plugins -# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) +# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: enabled: - go@1.23.3 - node@18.20.5 - python@3.10.8 -# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) +# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration) lint: enabled: - actionlint@1.7.5 From 2e7ac26e4084b54350a76ef24e89e277edc857f8 Mon Sep 17 00:00:00 2001 From: Ryan Fox-Tyler <60440289+ryanfoxtyler@users.noreply.github.com> Date: Fri, 3 Jan 2025 09:36:31 -0500 Subject: [PATCH 17/23] chore: move to Warp runners for GitHub Actions (#50) **Description** Move GitHub Actions workflows to Warp runners for faster tests through larger machine access. **Checklist** - [X] Code compiles correctly and linting passes locally --- .github/actionlint.yml | 5 +++++ .github/pull_request_template.md | 2 -- .github/workflows/ci-go-lint.yaml | 3 ++- .github/workflows/ci-go-tests.yaml | 3 ++- .trunk/configs/.markdownlint.json | 3 ++- README.md | 1 - 6 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .github/actionlint.yml diff --git a/.github/actionlint.yml b/.github/actionlint.yml new file mode 100644 index 0000000..5484e83 --- /dev/null +++ b/.github/actionlint.yml @@ -0,0 +1,5 @@ +self-hosted-runner: + # Labels of self-hosted runner in array of string + labels: + - warp-ubuntu-latest-arm64-2x + - warp-ubuntu-latest-arm64-4x diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e62dfbf..76974d6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,3 @@ - - **Description** Please explain the changes you made here. diff --git a/.github/workflows/ci-go-lint.yaml b/.github/workflows/ci-go-lint.yaml index 14cb093..be45df1 100644 --- a/.github/workflows/ci-go-lint.yaml +++ b/.github/workflows/ci-go-lint.yaml @@ -10,6 +10,7 @@ on: paths: - "**/*.go" - "**/go.mod" + - .github/workflows/* permissions: contents: read @@ -17,7 +18,7 @@ permissions: jobs: ci-go-lint: - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-arm64-2x steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/ci-go-tests.yaml b/.github/workflows/ci-go-tests.yaml index a0dc02d..69052ed 100644 --- a/.github/workflows/ci-go-tests.yaml +++ b/.github/workflows/ci-go-tests.yaml @@ -10,6 +10,7 @@ on: paths: - "**/*.go" - "**/go.mod" + - .github/workflows/* permissions: contents: read @@ -17,7 +18,7 @@ permissions: jobs: ci-go-tests: - runs-on: ubuntu-24.04 + runs-on: warp-ubuntu-latest-arm64-4x steps: - uses: actions/checkout@v4 diff --git a/.trunk/configs/.markdownlint.json b/.trunk/configs/.markdownlint.json index a7c5d5c..449148d 100644 --- a/.trunk/configs/.markdownlint.json +++ b/.trunk/configs/.markdownlint.json @@ -3,5 +3,6 @@ "no-inline-html": false, "no-bare-urls": false, "no-space-in-emphasis": false, - "no-emphasis-as-heading": false + "no-emphasis-as-heading": false, + "first-line-heading": false } diff --git a/README.md b/README.md index 3c9493b..6251714 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -

[![modus](https://github.com/user-attachments/assets/1a6020bd-d041-4dd0-b4a9-ce01dc015b65)](https://github.com/hypermodeinc/modusdb) From bcd42e1bbcb3b100e45386eb22c071c4b2833026 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Fri, 3 Jan 2025 08:25:39 -0800 Subject: [PATCH 18/23] feat: add readfrom json tag to support reverse edges (#49) **Description** This PR adds support for reverse edges via the readFrom json tag **Checklist** - [x] Code compiles correctly and linting passes locally - [x] For all _code_ changes, an entry added to the `CHANGELOG.md` file describing and linking to this PR - [x] Tests added for new functionality, or regression tests for bug fixes added as applicable - [ ] For public APIs, new features, etc., PR on [docs repo](https://github.com/hypermodeinc/docs) staged and linked here --- CHANGELOG.md | 5 ++ api.go | 2 +- api_dql.go | 16 +++-- api_mutate_helper.go | 37 ++++++++--- api_query_helper.go | 20 ++---- api_reflect.go | 54 ++++++++++++---- api_test.go | 149 +++++++++++++++++++++++++++++++++++++++++-- 7 files changed, 236 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2999085..4d92b85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## UNRELEASED + +- feat: add readfrom json tag to support reverse edges + [#49](https://github.com/hypermodeinc/modusDB/pull/49) + ## 2025-01-02 - Version 0.1.0 Baseline for the changelog. diff --git a/api.go b/api.go index 379fccd..652ea19 100644 --- a/api.go +++ b/api.go @@ -155,7 +155,7 @@ func Query[T any](db *DB, queryParams QueryParams, ns ...uint64) ([]uint64, []T, return nil, nil, err } - return executeQuery[T](ctx, n, queryParams, false) + return executeQuery[T](ctx, n, queryParams, true) } func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, T, error) { diff --git a/api_dql.go b/api_dql.go index 203440a..6c63171 100644 --- a/api_dql.go +++ b/api_dql.go @@ -21,9 +21,9 @@ const ( objQuery = ` { obj(func: %s) { - uid + gid: uid expand(_all_) { - uid + gid: uid expand(_all_) dgraph.type } @@ -36,9 +36,9 @@ const ( objsQuery = ` { objs(func: type("%s")%s) @filter(%s) { - uid + gid: uid expand(_all_) { - uid + gid: uid expand(_all_) dgraph.type } @@ -48,6 +48,14 @@ const ( } ` + reverseEdgeQuery = ` + %s: ~%s { + gid: uid + expand(_all_) + dgraph.type + } + ` + funcUid = `uid(%d)` funcEq = `eq(%s, %s)` funcSimilarTo = `similar_to(%s, %d, "[%s]")` diff --git a/api_mutate_helper.go b/api_mutate_helper.go index 2dd4188..f90c258 100644 --- a/api_mutate_helper.go +++ b/api_mutate_helper.go @@ -13,6 +13,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/dql" @@ -39,18 +40,34 @@ func generateCreateDqlMutationsAndSchema[T any](ctx context.Context, n *Namespac nquads := make([]*api.NQuad, 0) uniqueConstraintFound := false for jsonName, value := range jsonTagToValue { + var val *api.Value + var valType pb.Posting_ValType + + reflectValueType := reflect.TypeOf(value) + var nquad *api.NQuad + if jsonToReverseEdgeTags[jsonName] != "" { + if reflectValueType.Kind() != reflect.Slice || reflectValueType.Elem().Kind() != reflect.Struct { + return fmt.Errorf("reverse edge %s should be a slice of structs", jsonName) + } + reverseEdge := jsonToReverseEdgeTags[jsonName] + typeName := strings.Split(reverseEdge, ".")[0] + u := &pb.SchemaUpdate{ + Predicate: addNamespace(n.id, reverseEdge), + ValueType: pb.Posting_UID, + Directive: pb.SchemaUpdate_REVERSE, + } + sch.Preds = append(sch.Preds, u) + sch.Types = append(sch.Types, &pb.TypeUpdate{ + TypeName: addNamespace(n.id, typeName), + Fields: []*pb.SchemaUpdate{u}, + }) continue } if jsonName == "gid" { uniqueConstraintFound = true continue } - var val *api.Value - var valType pb.Posting_ValType - - reflectValueType := reflect.TypeOf(value) - var nquad *api.NQuad if reflectValueType.Kind() == reflect.Struct { value = reflect.ValueOf(value).Interface() @@ -87,16 +104,18 @@ func generateCreateDqlMutationsAndSchema[T any](ctx context.Context, n *Namespac Predicate: getPredicateName(t.Name(), jsonName), } + u := &pb.SchemaUpdate{ + Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), + ValueType: valType, + } + if valType == pb.Posting_UID { nquad.ObjectId = fmt.Sprint(value) + u.Directive = pb.SchemaUpdate_REVERSE } else { nquad.ObjectValue = val } - u := &pb.SchemaUpdate{ - Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), - ValueType: valType, - } if jsonToDbTags[jsonName] != nil { constraint := jsonToDbTags[jsonName].constraint if constraint == "vector" && valType != pb.Posting_VFLOAT { diff --git a/api_query_helper.go b/api_query_helper.go index 5c2552f..9ec211c 100644 --- a/api_query_helper.go +++ b/api_query_helper.go @@ -54,13 +54,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac readFromQuery := "" if withReverse { for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { - readFromQuery += fmt.Sprintf(` - %s: ~%s { - uid - expand(_all_) - dgraph.type - } - `, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) + readFromQuery += fmt.Sprintf(reverseEdgeQuery, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) } } @@ -106,7 +100,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac // Map the dynamic struct to the final type T finalObject := reflect.New(t).Interface() - gid, err = mapDynamicToFinal(result.Obj[0], finalObject) + gid, err = mapDynamicToFinal(result.Obj[0], finalObject, false) if err != nil { return 0, obj, err } @@ -152,13 +146,7 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar readFromQuery := "" if withReverse { for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { - readFromQuery += fmt.Sprintf(` - %s: ~%s { - uid - expand(_all_) - dgraph.type - } - `, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) + readFromQuery += fmt.Sprintf(reverseEdgeQuery, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) } } @@ -197,7 +185,7 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar var objs []T for _, obj := range result.Objs { finalObject := reflect.New(t).Interface() - gid, err := mapDynamicToFinal(obj, finalObject) + gid, err := mapDynamicToFinal(obj, finalObject, false) if err != nil { return nil, nil, err } diff --git a/api_reflect.go b/api_reflect.go index 1779a81..5a32b01 100644 --- a/api_reflect.go +++ b/api_reflect.go @@ -82,7 +82,7 @@ func createDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, dept field, _ := t.FieldByName(fieldName) if fieldName != "Gid" { if field.Type.Kind() == reflect.Struct { - if depth <= 2 { + if depth <= 1 { nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type) nestedType := createDynamicStruct(field.Type, nestedFieldToJsonTags, depth+1) fields = append(fields, reflect.StructField{ @@ -100,6 +100,15 @@ func createDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, dept Type: reflect.PointerTo(nestedType), Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), }) + } else if field.Type.Kind() == reflect.Slice && + field.Type.Elem().Kind() == reflect.Struct { + nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type.Elem()) + nestedType := createDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: reflect.SliceOf(nestedType), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) } else { fields = append(fields, reflect.StructField{ Name: field.Name, @@ -111,9 +120,9 @@ func createDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, dept } } fields = append(fields, reflect.StructField{ - Name: "Uid", + Name: "Gid", Type: reflect.TypeOf(""), - Tag: reflect.StructTag(`json:"uid"`), + Tag: reflect.StructTag(`json:"gid"`), }, reflect.StructField{ Name: "DgraphType", Type: reflect.TypeOf([]string{}), @@ -122,7 +131,7 @@ func createDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, dept return reflect.StructOf(fields) } -func mapDynamicToFinal(dynamic any, final any) (uint64, error) { +func mapDynamicToFinal(dynamic any, final any, isNested bool) (uint64, error) { vFinal := reflect.ValueOf(final).Elem() vDynamic := reflect.ValueOf(dynamic).Elem() @@ -135,35 +144,54 @@ func mapDynamicToFinal(dynamic any, final any) (uint64, error) { dynamicValue := vDynamic.Field(i) var finalField reflect.Value - if dynamicField.Name == "Uid" { + if dynamicField.Name == "Gid" { finalField = vFinal.FieldByName("Gid") gidStr := dynamicValue.String() gid, _ = strconv.ParseUint(gidStr, 0, 64) } else if dynamicField.Name == "DgraphType" { - fieldArr := dynamicValue.Interface().([]string) - if len(fieldArr) == 0 { - return 0, ErrNoObjFound + fieldArrInterface := dynamicValue.Interface() + fieldArr, ok := fieldArrInterface.([]string) + if ok { + if len(fieldArr) == 0 { + if !isNested { + return 0, ErrNoObjFound + } else { + continue + } + } + } else { + return 0, fmt.Errorf("DgraphType field should be an array of strings") } } else { finalField = vFinal.FieldByName(dynamicField.Name) } if dynamicFieldType.Kind() == reflect.Struct { - _, err := mapDynamicToFinal(dynamicValue.Addr().Interface(), finalField.Addr().Interface()) + _, err := mapDynamicToFinal(dynamicValue.Addr().Interface(), finalField.Addr().Interface(), true) if err != nil { return 0, err } } else if dynamicFieldType.Kind() == reflect.Ptr && dynamicFieldType.Elem().Kind() == reflect.Struct { // if field is a pointer, find if the underlying is a struct - _, err := mapDynamicToFinal(dynamicValue.Interface(), finalField.Interface()) + _, err := mapDynamicToFinal(dynamicValue.Interface(), finalField.Interface(), true) if err != nil { return 0, err } - + } else if dynamicFieldType.Kind() == reflect.Slice && + dynamicFieldType.Elem().Kind() == reflect.Struct { + for j := 0; j < dynamicValue.Len(); j++ { + sliceElem := dynamicValue.Index(j).Addr().Interface() + finalSliceElem := reflect.New(finalField.Type().Elem()).Elem() + _, err := mapDynamicToFinal(sliceElem, finalSliceElem.Addr().Interface(), true) + if err != nil { + return 0, err + } + finalField.Set(reflect.Append(finalField, finalSliceElem)) + } } else { if finalField.IsValid() && finalField.CanSet() { - // if field name is uid, convert it to uint64 - if dynamicField.Name == "Uid" { + // if field name is gid, convert it to uint64 + if dynamicField.Name == "Gid" { finalField.SetUint(gid) } else { finalField.Set(dynamicValue) diff --git a/api_test.go b/api_test.go index 0ae4bea..f7f15e0 100644 --- a/api_test.go +++ b/api_test.go @@ -400,10 +400,10 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { } type Project struct { - Gid uint64 `json:"gid,omitempty"` - Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` - // Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` + Gid uint64 `json:"gid,omitempty"` + Name string `json:"name,omitempty"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` } type Branch struct { @@ -413,6 +413,147 @@ type Branch struct { Proj Project `json:"proj,omitempty"` } +func TestReverseEdgeGet(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + projGid, project, err := modusdb.Create(db, Project{ + Name: "P", + ClerkId: "456", + Branches: []Branch{ + {Name: "B", ClerkId: "123"}, + {Name: "B2", ClerkId: "456"}, + }, + }, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "P", project.Name) + require.Equal(t, project.Gid, projGid) + + // modifying a read-only field will be a no-op + require.Len(t, project.Branches, 0) + + branch1 := Branch{ + Name: "B", + ClerkId: "123", + Proj: Project{ + Gid: projGid, + }, + } + + branch1Gid, branch1, err := modusdb.Create(db, branch1, db1.ID()) + require.NoError(t, err) + + require.Equal(t, "B", branch1.Name) + require.Equal(t, branch1.Gid, branch1Gid) + require.Equal(t, projGid, branch1.Proj.Gid) + require.Equal(t, "P", branch1.Proj.Name) + + branch2 := Branch{ + Name: "B2", + ClerkId: "456", + Proj: Project{ + Gid: projGid, + }, + } + + branch2Gid, branch2, err := modusdb.Create(db, branch2, db1.ID()) + require.NoError(t, err) + require.Equal(t, "B2", branch2.Name) + require.Equal(t, branch2.Gid, branch2Gid) + require.Equal(t, projGid, branch2.Proj.Gid) + + getProjGid, queriedProject, err := modusdb.Get[Project](db, projGid, db1.ID()) + require.NoError(t, err) + require.Equal(t, projGid, getProjGid) + require.Equal(t, "P", queriedProject.Name) + require.Len(t, queriedProject.Branches, 2) + require.Equal(t, "B", queriedProject.Branches[0].Name) + require.Equal(t, "B2", queriedProject.Branches[1].Name) + + queryBranchesGids, queriedBranches, err := modusdb.Query[Branch](db, modusdb.QueryParams{}, db1.ID()) + require.NoError(t, err) + require.Len(t, queriedBranches, 2) + require.Len(t, queryBranchesGids, 2) + require.Equal(t, "B", queriedBranches[0].Name) + require.Equal(t, "B2", queriedBranches[1].Name) + + // max depth is 2, so we should not see the branches within project + require.Len(t, queriedBranches[0].Proj.Branches, 0) + + _, _, err = modusdb.Delete[Project](db, projGid, db1.ID()) + require.NoError(t, err) + + queryBranchesGids, queriedBranches, err = modusdb.Query[Branch](db, modusdb.QueryParams{}, db1.ID()) + require.NoError(t, err) + require.Len(t, queriedBranches, 2) + require.Len(t, queryBranchesGids, 2) + require.Equal(t, "B", queriedBranches[0].Name) + require.Equal(t, "B2", queriedBranches[1].Name) +} + +func TestReverseEdgeQuery(t *testing.T) { + ctx := context.Background() + db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer db.Close() + + db1, err := db.CreateNamespace() + require.NoError(t, err) + + require.NoError(t, db1.DropData(ctx)) + + projects := []Project{ + {Name: "P1", ClerkId: "456"}, + {Name: "P2", ClerkId: "789"}, + } + + branchCounter := 1 + clerkCounter := 100 + + for _, project := range projects { + projGid, project, err := modusdb.Create(db, project, db1.ID()) + require.NoError(t, err) + require.Equal(t, project.Name, project.Name) + require.Equal(t, project.Gid, projGid) + + branches := []Branch{ + {Name: fmt.Sprintf("B%d", branchCounter), ClerkId: fmt.Sprintf("%d", clerkCounter), Proj: Project{Gid: projGid}}, + {Name: fmt.Sprintf("B%d", branchCounter+1), ClerkId: fmt.Sprintf("%d", clerkCounter+1), Proj: Project{Gid: projGid}}, + } + branchCounter += 2 + clerkCounter += 2 + + for _, branch := range branches { + branchGid, branch, err := modusdb.Create(db, branch, db1.ID()) + require.NoError(t, err) + require.Equal(t, branch.Name, branch.Name) + require.Equal(t, branch.Gid, branchGid) + require.Equal(t, projGid, branch.Proj.Gid) + } + } + + queriedProjectsGids, queriedProjects, err := modusdb.Query[Project](db, modusdb.QueryParams{}, db1.ID()) + require.NoError(t, err) + require.Len(t, queriedProjects, 2) + require.Len(t, queriedProjectsGids, 2) + require.Equal(t, "P1", queriedProjects[0].Name) + require.Equal(t, "P2", queriedProjects[1].Name) + require.Len(t, queriedProjects[0].Branches, 2) + require.Len(t, queriedProjects[1].Branches, 2) + require.Equal(t, "B1", queriedProjects[0].Branches[0].Name) + require.Equal(t, "B2", queriedProjects[0].Branches[1].Name) + require.Equal(t, "B3", queriedProjects[1].Branches[0].Name) + require.Equal(t, "B4", queriedProjects[1].Branches[1].Name) +} + func TestNestedObjectMutation(t *testing.T) { ctx := context.Background() db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) From dd2ea8084353aa8c268d4bf38bcc753be59851b4 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Fri, 3 Jan 2025 10:25:35 -0800 Subject: [PATCH 19/23] chore: Refactoring package management (#51) **Description** refactoring packages for clearer code, remove duplications, etc **Checklist** - [x] Code compiles correctly and linting passes locally - [x] For all _code_ changes, an entry added to the `CHANGELOG.md` file describing and linking to this PR - [ ] Tests added for new functionality, or regression tests for bug fixes added as applicable - [ ] For public APIs, new features, etc., PR on [docs repo](https://github.com/hypermodeinc/docs) staged and linked here --- CHANGELOG.md | 2 + api.go | 22 +- api/mutations/mutations.go | 71 +++++ api/query_gen/dql_query.go | 173 ++++++++++ api/utils/dgraph.go | 147 +++++++++ api/utils/reflect.go | 210 ++++++++++++ utils.go => api/utils/utils.go | 11 +- api_dql.go | 227 ------------- api_mutate_helper.go | 299 ------------------ api_mutation_gen.go | 121 +++++++ api_mutation_helpers.go | 123 +++++++ api_query_helper.go => api_query_execution.go | 81 +++-- api_reflect.go | 202 +----------- api_test.go | 5 +- api_types.go | 183 +++++------ 15 files changed, 985 insertions(+), 892 deletions(-) create mode 100644 api/mutations/mutations.go create mode 100644 api/query_gen/dql_query.go create mode 100644 api/utils/dgraph.go create mode 100644 api/utils/reflect.go rename utils.go => api/utils/utils.go (62%) delete mode 100644 api_dql.go delete mode 100644 api_mutate_helper.go create mode 100644 api_mutation_gen.go create mode 100644 api_mutation_helpers.go rename api_query_helper.go => api_query_execution.go (69%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d92b85..e92097c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - feat: add readfrom json tag to support reverse edges [#49](https://github.com/hypermodeinc/modusDB/pull/49) +- chore: Refactoring package management #51 [#51](https://github.com/hypermodeinc/modusDB/pull/51) + ## 2025-01-02 - Version 0.1.0 Baseline for the changelog. diff --git a/api.go b/api.go index 652ea19..beadf9f 100644 --- a/api.go +++ b/api.go @@ -14,6 +14,7 @@ import ( "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/schema" + "github.com/hypermodeinc/modusdb/api/utils" ) func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { @@ -34,7 +35,7 @@ func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateCreateDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) if err != nil { return 0, object, err } @@ -66,14 +67,14 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { return 0, object, false, err } - gid, cf, err := getUniqueConstraint[T](object) + gid, cf, err := GetUniqueConstraint[T](object) if err != nil { return 0, object, false, err } dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateCreateDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) if err != nil { return 0, object, false, err } @@ -83,19 +84,14 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { return 0, object, false, err } - if gid != 0 { - gid, _, err = getByGidWithObject[T](ctx, n, gid, object) - if err != nil && err != ErrNoObjFound { - return 0, object, false, err - } - wasFound = err == nil - } else if cf != nil { - gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, object) - if err != nil && err != ErrNoObjFound { + if gid != 0 || cf != nil { + gid, err = getExistingObject[T](ctx, n, gid, cf, object) + if err != nil && err != utils.ErrNoObjFound { return 0, object, false, err } wasFound = err == nil } + if gid == 0 { gid, err = db.z.nextUID() if err != nil { @@ -104,7 +100,7 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { } dms = make([]*dql.Mutation, 0) - err = generateCreateDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) if err != nil { return 0, object, false, err } diff --git a/api/mutations/mutations.go b/api/mutations/mutations.go new file mode 100644 index 0000000..4e4fedf --- /dev/null +++ b/api/mutations/mutations.go @@ -0,0 +1,71 @@ +package mutations + +import ( + "fmt" + "reflect" + "strings" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/hypermodeinc/modusdb/api/utils" +) + +func HandleReverseEdge(jsonName string, value reflect.Type, nsId uint64, sch *schema.ParsedSchema, + jsonToReverseEdgeTags map[string]string) error { + if jsonToReverseEdgeTags[jsonName] == "" { + return nil + } + + if value.Kind() != reflect.Slice || value.Elem().Kind() != reflect.Struct { + return fmt.Errorf("reverse edge %s should be a slice of structs", jsonName) + } + + reverseEdge := jsonToReverseEdgeTags[jsonName] + typeName := strings.Split(reverseEdge, ".")[0] + u := &pb.SchemaUpdate{ + Predicate: utils.AddNamespace(nsId, reverseEdge), + ValueType: pb.Posting_UID, + Directive: pb.SchemaUpdate_REVERSE, + } + + sch.Preds = append(sch.Preds, u) + sch.Types = append(sch.Types, &pb.TypeUpdate{ + TypeName: utils.AddNamespace(nsId, typeName), + Fields: []*pb.SchemaUpdate{u}, + }) + return nil +} + +func CreateNQuadAndSchema(value any, gid uint64, jsonName string, t reflect.Type, + nsId uint64) (*api.NQuad, *pb.SchemaUpdate, error) { + valType, err := utils.ValueToPosting_ValType(value) + if err != nil { + return nil, nil, err + } + + val, err := utils.ValueToApiVal(value) + if err != nil { + return nil, nil, err + } + + nquad := &api.NQuad{ + Namespace: nsId, + Subject: fmt.Sprint(gid), + Predicate: utils.GetPredicateName(t.Name(), jsonName), + } + + u := &pb.SchemaUpdate{ + Predicate: utils.AddNamespace(nsId, utils.GetPredicateName(t.Name(), jsonName)), + ValueType: valType, + } + + if valType == pb.Posting_UID { + nquad.ObjectId = fmt.Sprint(value) + u.Directive = pb.SchemaUpdate_REVERSE + } else { + nquad.ObjectValue = val + } + + return nquad, u, nil +} diff --git a/api/query_gen/dql_query.go b/api/query_gen/dql_query.go new file mode 100644 index 0000000..5922342 --- /dev/null +++ b/api/query_gen/dql_query.go @@ -0,0 +1,173 @@ +package query_gen + +import ( + "fmt" + "strconv" + "strings" +) + +type QueryFunc func() string + +const ( + ObjQuery = ` + { + obj(func: %s) { + gid: uid + expand(_all_) { + gid: uid + expand(_all_) + dgraph.type + } + dgraph.type + %s + } + } + ` + + ObjsQuery = ` + { + objs(func: type("%s")%s) @filter(%s) { + gid: uid + expand(_all_) { + gid: uid + expand(_all_) + dgraph.type + } + dgraph.type + %s + } + } + ` + + ReverseEdgeQuery = ` + %s: ~%s { + gid: uid + expand(_all_) + dgraph.type + } + ` + + FuncUid = `uid(%d)` + FuncEq = `eq(%s, %s)` + FuncSimilarTo = `similar_to(%s, %d, "[%s]")` + FuncAllOfTerms = `allofterms(%s, "%s")` + FuncAnyOfTerms = `anyofterms(%s, "%s")` + FuncAllOfText = `alloftext(%s, "%s")` + FuncAnyOfText = `anyoftext(%s, "%s")` + FuncRegExp = `regexp(%s, /%s/)` + FuncLe = `le(%s, %s)` + FuncGe = `ge(%s, %s)` + FuncGt = `gt(%s, %s)` + FuncLt = `lt(%s, %s)` +) + +func BuildUidQuery(gid uint64) QueryFunc { + return func() string { + return fmt.Sprintf(FuncUid, gid) + } +} + +func BuildEqQuery(key string, value any) QueryFunc { + return func() string { + return fmt.Sprintf(FuncEq, key, value) + } +} + +func BuildSimilarToQuery(indexAttr string, topK int64, vec []float32) QueryFunc { + vecStrArr := make([]string, len(vec)) + for i := range vec { + vecStrArr[i] = strconv.FormatFloat(float64(vec[i]), 'f', -1, 32) + } + vecStr := strings.Join(vecStrArr, ",") + return func() string { + return fmt.Sprintf(FuncSimilarTo, indexAttr, topK, vecStr) + } +} + +func BuildAllOfTermsQuery(attr string, terms string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncAllOfTerms, attr, terms) + } +} + +func BuildAnyOfTermsQuery(attr string, terms string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncAnyOfTerms, attr, terms) + } +} + +func BuildAllOfTextQuery(attr, text string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncAllOfText, attr, text) + } +} + +func BuildAnyOfTextQuery(attr, text string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncAnyOfText, attr, text) + } +} + +func BuildRegExpQuery(attr, pattern string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncRegExp, attr, pattern) + } +} + +func BuildLeQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncLe, attr, value) + } +} + +func BuildGeQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncGe, attr, value) + } +} + +func BuildGtQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncGt, attr, value) + } +} + +func BuildLtQuery(attr, value string) QueryFunc { + return func() string { + return fmt.Sprintf(FuncLt, attr, value) + } +} + +func And(qfs ...QueryFunc) QueryFunc { + return func() string { + qs := make([]string, len(qfs)) + for i, qf := range qfs { + qs[i] = qf() + } + return strings.Join(qs, " AND ") + } +} + +func Or(qfs ...QueryFunc) QueryFunc { + return func() string { + qs := make([]string, len(qfs)) + for i, qf := range qfs { + qs[i] = qf() + } + return strings.Join(qs, " OR ") + } +} + +func Not(qf QueryFunc) QueryFunc { + return func() string { + return "NOT " + qf() + } +} + +func FormatObjQuery(qf QueryFunc, extraFields string) string { + return fmt.Sprintf(ObjQuery, qf(), extraFields) +} + +func FormatObjsQuery(typeName string, qf QueryFunc, paginationAndSorting string, extraFields string) string { + return fmt.Sprintf(ObjsQuery, typeName, paginationAndSorting, qf(), extraFields) +} diff --git a/api/utils/dgraph.go b/api/utils/dgraph.go new file mode 100644 index 0000000..fc49050 --- /dev/null +++ b/api/utils/dgraph.go @@ -0,0 +1,147 @@ +package utils + +import ( + "encoding/binary" + "fmt" + "time" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/types" + "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/wkb" +) + +func addIndex(u *pb.SchemaUpdate, index string, uniqueConstraintExists bool) bool { + u.Directive = pb.SchemaUpdate_INDEX + switch index { + case "exact": + u.Tokenizer = []string{"exact"} + case "term": + u.Tokenizer = []string{"term"} + case "hash": + u.Tokenizer = []string{"hash"} + case "unique": + u.Tokenizer = []string{"exact"} + u.Unique = true + u.Upsert = true + uniqueConstraintExists = true + case "fulltext": + u.Tokenizer = []string{"fulltext"} + case "trigram": + u.Tokenizer = []string{"trigram"} + case "vector": + u.IndexSpecs = []*pb.VectorIndexSpec{ + { + Name: "hnsw", + Options: []*pb.OptionPair{ + { + Key: "metric", + Value: "cosine", + }, + }, + }, + } + default: + return uniqueConstraintExists + } + return uniqueConstraintExists +} + +func ValueToPosting_ValType(v any) (pb.Posting_ValType, error) { + switch v.(type) { + case string: + return pb.Posting_STRING, nil + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32: + return pb.Posting_INT, nil + case uint64: + return pb.Posting_UID, nil + case bool: + return pb.Posting_BOOL, nil + case float32, float64: + return pb.Posting_FLOAT, nil + case []byte: + return pb.Posting_BINARY, nil + case time.Time: + return pb.Posting_DATETIME, nil + case geom.Point: + return pb.Posting_GEO, nil + case []float32, []float64: + return pb.Posting_VFLOAT, nil + default: + return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) + } +} + +func ValueToApiVal(v any) (*api.Value, error) { + switch val := v.(type) { + case string: + return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil + case int: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case int64: + return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil + case uint8: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint16: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint32: + return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil + case uint64: + return &api.Value{Val: &api.Value_UidVal{UidVal: val}}, nil + case bool: + return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil + case float32: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil + case float64: + return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil + case []float32: + return &api.Value{Val: &api.Value_Vfloat32Val{ + Vfloat32Val: types.FloatArrayAsBytes(val)}}, nil + case []float64: + float32Slice := make([]float32, len(val)) + for i, v := range val { + float32Slice[i] = float32(v) + } + return &api.Value{Val: &api.Value_Vfloat32Val{ + Vfloat32Val: types.FloatArrayAsBytes(float32Slice)}}, nil + case []byte: + return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil + case time.Time: + bytes, err := val.MarshalBinary() + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil + case geom.Point: + bytes, err := wkb.Marshal(&val, binary.LittleEndian) + if err != nil { + return nil, err + } + return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil + case uint: + return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil + default: + return nil, fmt.Errorf("unsupported type %T", v) + } +} + +func HandleConstraints(u *pb.SchemaUpdate, jsonToDbTags map[string]*DbTag, jsonName string, + valType pb.Posting_ValType, uniqueConstraintFound bool) (bool, error) { + if jsonToDbTags[jsonName] == nil { + return uniqueConstraintFound, nil + } + + constraint := jsonToDbTags[jsonName].Constraint + if constraint == "vector" && valType != pb.Posting_VFLOAT { + return false, fmt.Errorf("vector index can only be applied to []float values") + } + + return addIndex(u, constraint, uniqueConstraintFound), nil +} diff --git a/api/utils/reflect.go b/api/utils/reflect.go new file mode 100644 index 0000000..a4a57e0 --- /dev/null +++ b/api/utils/reflect.go @@ -0,0 +1,210 @@ +package utils + +import ( + "fmt" + "reflect" + "strconv" + "strings" +) + +type DbTag struct { + Constraint string +} + +func GetFieldTags(t reflect.Type) (fieldToJsonTags map[string]string, + jsonToDbTags map[string]*DbTag, jsonToReverseEdgeTags map[string]string, err error) { + + fieldToJsonTags = make(map[string]string) + jsonToDbTags = make(map[string]*DbTag) + jsonToReverseEdgeTags = make(map[string]string) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + return nil, nil, nil, fmt.Errorf("field %s has no json tag", field.Name) + } + jsonName := strings.Split(jsonTag, ",")[0] + fieldToJsonTags[field.Name] = jsonName + + reverseEdgeTag := field.Tag.Get("readFrom") + if reverseEdgeTag != "" { + typeAndField := strings.Split(reverseEdgeTag, ",") + if len(typeAndField) != 2 { + return nil, nil, nil, fmt.Errorf(`field %s has invalid readFrom tag, + expected format is type=,field=`, field.Name) + } + t := strings.Split(typeAndField[0], "=")[1] + f := strings.Split(typeAndField[1], "=")[1] + jsonToReverseEdgeTags[jsonName] = GetPredicateName(t, f) + } + + dbConstraintsTag := field.Tag.Get("db") + if dbConstraintsTag != "" { + jsonToDbTags[jsonName] = &DbTag{} + dbTagsSplit := strings.Split(dbConstraintsTag, ",") + for _, dbTag := range dbTagsSplit { + split := strings.Split(dbTag, "=") + if split[0] == "constraint" { + jsonToDbTags[jsonName].Constraint = split[1] + } + } + } + } + return fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, nil +} + +func GetJsonTagToValues(object any, fieldToJsonTags map[string]string) map[string]any { + values := make(map[string]any) + v := reflect.ValueOf(object) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + for fieldName, jsonName := range fieldToJsonTags { + fieldValue := v.FieldByName(fieldName) + values[jsonName] = fieldValue.Interface() + + } + return values +} + +func CreateDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, depth int) reflect.Type { + fields := make([]reflect.StructField, 0, len(fieldToJsonTags)) + for fieldName, jsonName := range fieldToJsonTags { + field, _ := t.FieldByName(fieldName) + if fieldName != "Gid" { + if field.Type.Kind() == reflect.Struct { + if depth <= 1 { + nestedFieldToJsonTags, _, _, _ := GetFieldTags(field.Type) + nestedType := CreateDynamicStruct(field.Type, nestedFieldToJsonTags, depth+1) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: nestedType, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } + } else if field.Type.Kind() == reflect.Ptr && + field.Type.Elem().Kind() == reflect.Struct { + nestedFieldToJsonTags, _, _, _ := GetFieldTags(field.Type.Elem()) + nestedType := CreateDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: reflect.PointerTo(nestedType), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } else if field.Type.Kind() == reflect.Slice && + field.Type.Elem().Kind() == reflect.Struct { + nestedFieldToJsonTags, _, _, _ := GetFieldTags(field.Type.Elem()) + nestedType := CreateDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: reflect.SliceOf(nestedType), + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } else { + fields = append(fields, reflect.StructField{ + Name: field.Name, + Type: field.Type, + Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), + }) + } + + } + } + fields = append(fields, reflect.StructField{ + Name: "Gid", + Type: reflect.TypeOf(""), + Tag: reflect.StructTag(`json:"gid"`), + }, reflect.StructField{ + Name: "DgraphType", + Type: reflect.TypeOf([]string{}), + Tag: reflect.StructTag(`json:"dgraph.type"`), + }) + return reflect.StructOf(fields) +} + +func MapDynamicToFinal(dynamic any, final any, isNested bool) (uint64, error) { + vFinal := reflect.ValueOf(final).Elem() + vDynamic := reflect.ValueOf(dynamic).Elem() + + gid := uint64(0) + + for i := 0; i < vDynamic.NumField(); i++ { + + dynamicField := vDynamic.Type().Field(i) + dynamicFieldType := dynamicField.Type + dynamicValue := vDynamic.Field(i) + + var finalField reflect.Value + if dynamicField.Name == "Gid" { + finalField = vFinal.FieldByName("Gid") + gidStr := dynamicValue.String() + gid, _ = strconv.ParseUint(gidStr, 0, 64) + } else if dynamicField.Name == "DgraphType" { + fieldArrInterface := dynamicValue.Interface() + fieldArr, ok := fieldArrInterface.([]string) + if ok { + if len(fieldArr) == 0 { + if !isNested { + return 0, ErrNoObjFound + } else { + continue + } + } + } else { + return 0, fmt.Errorf("DgraphType field should be an array of strings") + } + } else { + finalField = vFinal.FieldByName(dynamicField.Name) + } + if dynamicFieldType.Kind() == reflect.Struct { + _, err := MapDynamicToFinal(dynamicValue.Addr().Interface(), finalField.Addr().Interface(), true) + if err != nil { + return 0, err + } + } else if dynamicFieldType.Kind() == reflect.Ptr && + dynamicFieldType.Elem().Kind() == reflect.Struct { + // if field is a pointer, find if the underlying is a struct + _, err := MapDynamicToFinal(dynamicValue.Interface(), finalField.Interface(), true) + if err != nil { + return 0, err + } + } else if dynamicFieldType.Kind() == reflect.Slice && + dynamicFieldType.Elem().Kind() == reflect.Struct { + for j := 0; j < dynamicValue.Len(); j++ { + sliceElem := dynamicValue.Index(j).Addr().Interface() + finalSliceElem := reflect.New(finalField.Type().Elem()).Elem() + _, err := MapDynamicToFinal(sliceElem, finalSliceElem.Addr().Interface(), true) + if err != nil { + return 0, err + } + finalField.Set(reflect.Append(finalField, finalSliceElem)) + } + } else { + if finalField.IsValid() && finalField.CanSet() { + // if field name is gid, convert it to uint64 + if dynamicField.Name == "Gid" { + finalField.SetUint(gid) + } else { + finalField.Set(dynamicValue) + } + } + } + } + return gid, nil +} + +func ConvertDynamicToTyped[T any](obj any, t reflect.Type) (uint64, T, error) { + var result T + finalObject := reflect.New(t).Interface() + gid, err := MapDynamicToFinal(obj, finalObject, false) + if err != nil { + return 0, result, err + } + + if typedPtr, ok := finalObject.(*T); ok { + return gid, *typedPtr, nil + } else if dirType, ok := finalObject.(T); ok { + return gid, dirType, nil + } + return 0, result, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) +} diff --git a/utils.go b/api/utils/utils.go similarity index 62% rename from utils.go rename to api/utils/utils.go index e571bb2..ba726ed 100644 --- a/utils.go +++ b/api/utils/utils.go @@ -7,7 +7,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package modusdb +package utils import ( "fmt" @@ -15,10 +15,15 @@ import ( "github.com/dgraph-io/dgraph/v24/x" ) -func getPredicateName(typeName, fieldName string) string { +var ( + ErrNoObjFound = fmt.Errorf("no object found") + NoUniqueConstr = "unique constraint not defined for any field on type %s" +) + +func GetPredicateName(typeName, fieldName string) string { return fmt.Sprint(typeName, ".", fieldName) } -func addNamespace(ns uint64, pred string) string { +func AddNamespace(ns uint64, pred string) string { return x.NamespaceAttr(ns, pred) } diff --git a/api_dql.go b/api_dql.go deleted file mode 100644 index 6c63171..0000000 --- a/api_dql.go +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright 2025 Hypermode Inc. - * Licensed under the terms of the Apache License, Version 2.0 - * See the LICENSE file that accompanied this code for further details. - * - * SPDX-FileCopyrightText: 2025 Hypermode Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package modusdb - -import ( - "fmt" - "strconv" - "strings" -) - -type QueryFunc func() string - -const ( - objQuery = ` - { - obj(func: %s) { - gid: uid - expand(_all_) { - gid: uid - expand(_all_) - dgraph.type - } - dgraph.type - %s - } - } - ` - - objsQuery = ` - { - objs(func: type("%s")%s) @filter(%s) { - gid: uid - expand(_all_) { - gid: uid - expand(_all_) - dgraph.type - } - dgraph.type - %s - } - } - ` - - reverseEdgeQuery = ` - %s: ~%s { - gid: uid - expand(_all_) - dgraph.type - } - ` - - funcUid = `uid(%d)` - funcEq = `eq(%s, %s)` - funcSimilarTo = `similar_to(%s, %d, "[%s]")` - funcAllOfTerms = `allofterms(%s, "%s")` - funcAnyOfTerms = `anyofterms(%s, "%s")` - funcAllOfText = `alloftext(%s, "%s")` - funcAnyOfText = `anyoftext(%s, "%s")` - funcRegExp = `regexp(%s, /%s/)` - funcLe = `le(%s, %s)` - funcGe = `ge(%s, %s)` - funcGt = `gt(%s, %s)` - funcLt = `lt(%s, %s)` -) - -func buildUidQuery(gid uint64) QueryFunc { - return func() string { - return fmt.Sprintf(funcUid, gid) - } -} - -func buildEqQuery(key string, value any) QueryFunc { - return func() string { - return fmt.Sprintf(funcEq, key, value) - } -} - -func buildSimilarToQuery(indexAttr string, topK int64, vec []float32) QueryFunc { - vecStrArr := make([]string, len(vec)) - for i := range vec { - vecStrArr[i] = strconv.FormatFloat(float64(vec[i]), 'f', -1, 32) - } - vecStr := strings.Join(vecStrArr, ",") - return func() string { - return fmt.Sprintf(funcSimilarTo, indexAttr, topK, vecStr) - } -} - -func buildAllOfTermsQuery(attr string, terms string) QueryFunc { - return func() string { - return fmt.Sprintf(funcAllOfTerms, attr, terms) - } -} - -func buildAnyOfTermsQuery(attr string, terms string) QueryFunc { - return func() string { - return fmt.Sprintf(funcAnyOfTerms, attr, terms) - } -} - -func buildAllOfTextQuery(attr, text string) QueryFunc { - return func() string { - return fmt.Sprintf(funcAllOfText, attr, text) - } -} - -func buildAnyOfTextQuery(attr, text string) QueryFunc { - return func() string { - return fmt.Sprintf(funcAnyOfText, attr, text) - } -} - -func buildRegExpQuery(attr, pattern string) QueryFunc { - return func() string { - return fmt.Sprintf(funcRegExp, attr, pattern) - } -} - -func buildLeQuery(attr, value string) QueryFunc { - return func() string { - return fmt.Sprintf(funcLe, attr, value) - } -} - -func buildGeQuery(attr, value string) QueryFunc { - return func() string { - return fmt.Sprintf(funcGe, attr, value) - } -} - -func buildGtQuery(attr, value string) QueryFunc { - return func() string { - return fmt.Sprintf(funcGt, attr, value) - } -} - -func buildLtQuery(attr, value string) QueryFunc { - return func() string { - return fmt.Sprintf(funcLt, attr, value) - } -} - -func And(qfs ...QueryFunc) QueryFunc { - return func() string { - qs := make([]string, len(qfs)) - for i, qf := range qfs { - qs[i] = qf() - } - return strings.Join(qs, " AND ") - } -} - -func Or(qfs ...QueryFunc) QueryFunc { - return func() string { - qs := make([]string, len(qfs)) - for i, qf := range qfs { - qs[i] = qf() - } - return strings.Join(qs, " OR ") - } -} - -func Not(qf QueryFunc) QueryFunc { - return func() string { - return "NOT " + qf() - } -} - -func formatObjQuery(qf QueryFunc, extraFields string) string { - return fmt.Sprintf(objQuery, qf(), extraFields) -} - -func formatObjsQuery(typeName string, qf QueryFunc, paginationAndSorting string, extraFields string) string { - return fmt.Sprintf(objsQuery, typeName, paginationAndSorting, qf(), extraFields) -} - -// Helper function to combine multiple filters -func filtersToQueryFunc(typeName string, filter Filter) QueryFunc { - return filterToQueryFunc(typeName, filter) -} - -func paginationToQueryString(p Pagination) string { - paginationStr := "" - if p.Limit > 0 { - paginationStr += ", " + fmt.Sprintf("first: %d", p.Limit) - } - if p.Offset > 0 { - paginationStr += ", " + fmt.Sprintf("offset: %d", p.Offset) - } else if p.After != "" { - paginationStr += ", " + fmt.Sprintf("after: %s", p.After) - } - if paginationStr == "" { - return "" - } - return paginationStr -} - -func sortingToQueryString(typeName string, s Sorting) string { - if s.OrderAscField == "" && s.OrderDescField == "" { - return "" - } - - var parts []string - first, second := s.OrderDescField, s.OrderAscField - firstOp, secondOp := "orderdesc", "orderasc" - - if !s.OrderDescFirst { - first, second = s.OrderAscField, s.OrderDescField - firstOp, secondOp = "orderasc", "orderdesc" - } - - if first != "" { - parts = append(parts, fmt.Sprintf("%s: %s", firstOp, getPredicateName(typeName, first))) - } - if second != "" { - parts = append(parts, fmt.Sprintf("%s: %s", secondOp, getPredicateName(typeName, second))) - } - - return ", " + strings.Join(parts, ", ") -} diff --git a/api_mutate_helper.go b/api_mutate_helper.go deleted file mode 100644 index f90c258..0000000 --- a/api_mutate_helper.go +++ /dev/null @@ -1,299 +0,0 @@ -/* - * Copyright 2025 Hypermode Inc. - * Licensed under the terms of the Apache License, Version 2.0 - * See the LICENSE file that accompanied this code for further details. - * - * SPDX-FileCopyrightText: 2025 Hypermode Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package modusdb - -import ( - "context" - "fmt" - "reflect" - "strings" - - "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/dgraph-io/dgraph/v24/dql" - "github.com/dgraph-io/dgraph/v24/protos/pb" - "github.com/dgraph-io/dgraph/v24/query" - "github.com/dgraph-io/dgraph/v24/schema" - "github.com/dgraph-io/dgraph/v24/worker" - "github.com/dgraph-io/dgraph/v24/x" -) - -func generateCreateDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, - gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { - t := reflect.TypeOf(object) - if t.Kind() != reflect.Struct { - return fmt.Errorf("expected struct, got %s", t.Kind()) - } - - fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, err := getFieldTags(t) - if err != nil { - return err - } - jsonTagToValue := getJsonTagToValues(object, fieldToJsonTags) - - nquads := make([]*api.NQuad, 0) - uniqueConstraintFound := false - for jsonName, value := range jsonTagToValue { - var val *api.Value - var valType pb.Posting_ValType - - reflectValueType := reflect.TypeOf(value) - var nquad *api.NQuad - - if jsonToReverseEdgeTags[jsonName] != "" { - if reflectValueType.Kind() != reflect.Slice || reflectValueType.Elem().Kind() != reflect.Struct { - return fmt.Errorf("reverse edge %s should be a slice of structs", jsonName) - } - reverseEdge := jsonToReverseEdgeTags[jsonName] - typeName := strings.Split(reverseEdge, ".")[0] - u := &pb.SchemaUpdate{ - Predicate: addNamespace(n.id, reverseEdge), - ValueType: pb.Posting_UID, - Directive: pb.SchemaUpdate_REVERSE, - } - sch.Preds = append(sch.Preds, u) - sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: addNamespace(n.id, typeName), - Fields: []*pb.SchemaUpdate{u}, - }) - continue - } - if jsonName == "gid" { - uniqueConstraintFound = true - continue - } - - if reflectValueType.Kind() == reflect.Struct { - value = reflect.ValueOf(value).Interface() - newGid, err := getUidOrMutate(ctx, n.db, n, value) - if err != nil { - return err - } - value = newGid - } else if reflectValueType.Kind() == reflect.Pointer { - // dereference the pointer - reflectValueType = reflectValueType.Elem() - if reflectValueType.Kind() == reflect.Struct { - // convert value to pointer, and then dereference - value = reflect.ValueOf(value).Elem().Interface() - newGid, err := getUidOrMutate(ctx, n.db, n, value) - if err != nil { - return err - } - value = newGid - } - } - valType, err = valueToPosting_ValType(value) - if err != nil { - return err - } - val, err = valueToApiVal(value) - if err != nil { - return err - } - - nquad = &api.NQuad{ - Namespace: n.ID(), - Subject: fmt.Sprint(gid), - Predicate: getPredicateName(t.Name(), jsonName), - } - - u := &pb.SchemaUpdate{ - Predicate: addNamespace(n.id, getPredicateName(t.Name(), jsonName)), - ValueType: valType, - } - - if valType == pb.Posting_UID { - nquad.ObjectId = fmt.Sprint(value) - u.Directive = pb.SchemaUpdate_REVERSE - } else { - nquad.ObjectValue = val - } - - if jsonToDbTags[jsonName] != nil { - constraint := jsonToDbTags[jsonName].constraint - if constraint == "vector" && valType != pb.Posting_VFLOAT { - return fmt.Errorf("vector index can only be applied to []float values") - } - uniqueConstraintFound = addIndex(u, constraint, uniqueConstraintFound) - } - - sch.Preds = append(sch.Preds, u) - nquads = append(nquads, nquad) - } - if !uniqueConstraintFound { - return fmt.Errorf(NoUniqueConstr, t.Name()) - } - sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: addNamespace(n.id, t.Name()), - Fields: sch.Preds, - }) - - val, err := valueToApiVal(t.Name()) - if err != nil { - return err - } - typeNquad := &api.NQuad{ - Namespace: n.ID(), - Subject: fmt.Sprint(gid), - Predicate: "dgraph.type", - ObjectValue: val, - } - nquads = append(nquads, typeNquad) - - *dms = append(*dms, &dql.Mutation{ - Set: nquads, - }) - - return nil -} - -func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { - return []*dql.Mutation{{ - Del: []*api.NQuad{ - { - Namespace: n.ID(), - Subject: fmt.Sprint(gid), - Predicate: x.Star, - ObjectValue: &api.Value{ - Val: &api.Value_DefaultVal{DefaultVal: x.Star}, - }, - }, - }, - }} -} - -func applyDqlMutations(ctx context.Context, db *DB, dms []*dql.Mutation) error { - edges, err := query.ToDirectedEdges(dms, nil) - if err != nil { - return err - } - - if !db.isOpen { - return ErrClosedDB - } - - startTs, err := db.z.nextTs() - if err != nil { - return err - } - commitTs, err := db.z.nextTs() - if err != nil { - return err - } - - m := &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Edges: edges, - } - m.Edges, err = query.ExpandEdges(ctx, m) - if err != nil { - return fmt.Errorf("error expanding edges: %w", err) - } - - p := &pb.Proposal{Mutations: m, StartTs: startTs} - if err := worker.ApplyMutations(ctx, p); err != nil { - return err - } - - return worker.ApplyCommited(ctx, &pb.OracleDelta{ - Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, - }) -} - -func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) (uint64, error) { - gid, cf, err := getUniqueConstraint[T](object) - if err != nil { - return 0, err - } - - dms := make([]*dql.Mutation, 0) - sch := &schema.ParsedSchema{} - err = generateCreateDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) - if err != nil { - return 0, err - } - - err = n.alterSchemaWithParsed(ctx, sch) - if err != nil { - return 0, err - } - if gid != 0 { - gid, _, err = getByGidWithObject[T](ctx, n, gid, object) - if err != nil && err != ErrNoObjFound { - return 0, err - } - if err == nil { - return gid, nil - } - } else if cf != nil { - gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, object) - if err != nil && err != ErrNoObjFound { - return 0, err - } - if err == nil { - return gid, nil - } - } - - gid, err = db.z.nextUID() - if err != nil { - return 0, err - } - - dms = make([]*dql.Mutation, 0) - err = generateCreateDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) - if err != nil { - return 0, err - } - - err = applyDqlMutations(ctx, db, dms) - if err != nil { - return 0, err - } - - return gid, nil -} - -func addIndex(u *pb.SchemaUpdate, index string, uniqueConstraintExists bool) bool { - u.Directive = pb.SchemaUpdate_INDEX - switch index { - case "exact": - u.Tokenizer = []string{"exact"} - case "term": - u.Tokenizer = []string{"term"} - case "hash": - u.Tokenizer = []string{"hash"} - case "unique": - u.Tokenizer = []string{"exact"} - u.Unique = true - u.Upsert = true - uniqueConstraintExists = true - case "fulltext": - u.Tokenizer = []string{"fulltext"} - case "trigram": - u.Tokenizer = []string{"trigram"} - case "vector": - u.IndexSpecs = []*pb.VectorIndexSpec{ - { - Name: "hnsw", - Options: []*pb.OptionPair{ - { - Key: "metric", - Value: "cosine", - }, - }, - }, - } - default: - return uniqueConstraintExists - } - return uniqueConstraintExists -} diff --git a/api_mutation_gen.go b/api_mutation_gen.go new file mode 100644 index 0000000..3bfdc6f --- /dev/null +++ b/api_mutation_gen.go @@ -0,0 +1,121 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package modusdb + +import ( + "context" + "fmt" + "reflect" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/dql" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/x" + "github.com/hypermodeinc/modusdb/api/mutations" + "github.com/hypermodeinc/modusdb/api/utils" +) + +func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, + gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { + t := reflect.TypeOf(object) + if t.Kind() != reflect.Struct { + return fmt.Errorf("expected struct, got %s", t.Kind()) + } + + fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, err := utils.GetFieldTags(t) + if err != nil { + return err + } + jsonTagToValue := utils.GetJsonTagToValues(object, fieldToJsonTags) + + nquads := make([]*api.NQuad, 0) + uniqueConstraintFound := false + for jsonName, value := range jsonTagToValue { + + reflectValueType := reflect.TypeOf(value) + var nquad *api.NQuad + + if jsonToReverseEdgeTags[jsonName] != "" { + if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.id, sch, jsonToReverseEdgeTags); err != nil { + return err + } + continue + } + if jsonName == "gid" { + uniqueConstraintFound = true + continue + } + + value, err = processStructValue(ctx, value, n) + if err != nil { + return err + } + + value, err = processPointerValue(ctx, value, n) + if err != nil { + return err + } + + nquad, u, err := mutations.CreateNQuadAndSchema(value, gid, jsonName, t, n.ID()) + if err != nil { + return err + } + + uniqueConstraintFound, err = utils.HandleConstraints(u, jsonToDbTags, jsonName, u.ValueType, uniqueConstraintFound) + if err != nil { + return err + } + + sch.Preds = append(sch.Preds, u) + nquads = append(nquads, nquad) + } + if !uniqueConstraintFound { + return fmt.Errorf(utils.NoUniqueConstr, t.Name()) + } + + sch.Types = append(sch.Types, &pb.TypeUpdate{ + TypeName: utils.AddNamespace(n.id, t.Name()), + Fields: sch.Preds, + }) + + val, err := utils.ValueToApiVal(t.Name()) + if err != nil { + return err + } + typeNquad := &api.NQuad{ + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: "dgraph.type", + ObjectValue: val, + } + nquads = append(nquads, typeNquad) + + *dms = append(*dms, &dql.Mutation{ + Set: nquads, + }) + + return nil +} + +func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { + return []*dql.Mutation{{ + Del: []*api.NQuad{ + { + Namespace: n.ID(), + Subject: fmt.Sprint(gid), + Predicate: x.Star, + ObjectValue: &api.Value{ + Val: &api.Value_DefaultVal{DefaultVal: x.Star}, + }, + }, + }, + }} +} diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go new file mode 100644 index 0000000..d4b0cf4 --- /dev/null +++ b/api_mutation_helpers.go @@ -0,0 +1,123 @@ +package modusdb + +import ( + "context" + "fmt" + "reflect" + + "github.com/dgraph-io/dgraph/v24/dql" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/query" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/worker" + "github.com/hypermodeinc/modusdb/api/utils" +) + +func processStructValue(ctx context.Context, value any, n *Namespace) (any, error) { + if reflect.TypeOf(value).Kind() == reflect.Struct { + value = reflect.ValueOf(value).Interface() + newGid, err := getUidOrMutate(ctx, n.db, n, value) + if err != nil { + return nil, err + } + return newGid, nil + } + return value, nil +} + +func processPointerValue(ctx context.Context, value any, n *Namespace) (any, error) { + reflectValueType := reflect.TypeOf(value) + if reflectValueType.Kind() == reflect.Pointer { + reflectValueType = reflectValueType.Elem() + if reflectValueType.Kind() == reflect.Struct { + value = reflect.ValueOf(value).Elem().Interface() + return processStructValue(ctx, value, n) + } + } + return value, nil +} + +func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) (uint64, error) { + gid, cf, err := GetUniqueConstraint[T](object) + if err != nil { + return 0, err + } + + dms := make([]*dql.Mutation, 0) + sch := &schema.ParsedSchema{} + err = generateSetDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) + if err != nil { + return 0, err + } + + err = n.alterSchemaWithParsed(ctx, sch) + if err != nil { + return 0, err + } + if gid != 0 || cf != nil { + gid, err = getExistingObject(ctx, n, gid, cf, object) + if err != nil && err != utils.ErrNoObjFound { + return 0, err + } + if err == nil { + return gid, nil + } + } + + gid, err = db.z.nextUID() + if err != nil { + return 0, err + } + + dms = make([]*dql.Mutation, 0) + err = generateSetDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) + if err != nil { + return 0, err + } + + err = applyDqlMutations(ctx, db, dms) + if err != nil { + return 0, err + } + + return gid, nil +} + +func applyDqlMutations(ctx context.Context, db *DB, dms []*dql.Mutation) error { + edges, err := query.ToDirectedEdges(dms, nil) + if err != nil { + return err + } + + if !db.isOpen { + return ErrClosedDB + } + + startTs, err := db.z.nextTs() + if err != nil { + return err + } + commitTs, err := db.z.nextTs() + if err != nil { + return err + } + + m := &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Edges: edges, + } + m.Edges, err = query.ExpandEdges(ctx, m) + if err != nil { + return fmt.Errorf("error expanding edges: %w", err) + } + + p := &pb.Proposal{Mutations: m, StartTs: startTs} + if err := worker.ApplyMutations(ctx, p); err != nil { + return err + } + + return worker.ApplyCommited(ctx, &pb.OracleDelta{ + Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, + }) +} diff --git a/api_query_helper.go b/api_query_execution.go similarity index 69% rename from api_query_helper.go rename to api_query_execution.go index 9ec211c..bf9cb23 100644 --- a/api_query_helper.go +++ b/api_query_execution.go @@ -14,6 +14,9 @@ import ( "encoding/json" "fmt" "reflect" + + "github.com/hypermodeinc/modusdb/api/query_gen" + "github.com/hypermodeinc/modusdb/api/utils" ) func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, T, error) { @@ -47,14 +50,15 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac obj T, withReverse bool, args ...R) (uint64, T, error) { t := reflect.TypeOf(obj) - fieldToJsonTags, jsonToDbTag, jsonToReverseEdgeTags, err := getFieldTags(t) + fieldToJsonTags, jsonToDbTag, jsonToReverseEdgeTags, err := utils.GetFieldTags(t) if err != nil { return 0, obj, err } readFromQuery := "" if withReverse { for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { - readFromQuery += fmt.Sprintf(reverseEdgeQuery, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) + readFromQuery += fmt.Sprintf(query_gen.ReverseEdgeQuery, + utils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) } } @@ -62,14 +66,15 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac var query string gid, ok := any(args[0]).(uint64) if ok { - query = formatObjQuery(buildUidQuery(gid), readFromQuery) + query = query_gen.FormatObjQuery(query_gen.BuildUidQuery(gid), readFromQuery) } else if cf, ok = any(args[0]).(ConstrainedField); ok { - query = formatObjQuery(buildEqQuery(getPredicateName(t.Name(), cf.Key), cf.Value), readFromQuery) + query = query_gen.FormatObjQuery(query_gen.BuildEqQuery(utils.GetPredicateName(t.Name(), + cf.Key), cf.Value), readFromQuery) } else { return 0, obj, fmt.Errorf("invalid unique field type") } - if jsonToDbTag[cf.Key] != nil && jsonToDbTag[cf.Key].constraint == "" { + if jsonToDbTag[cf.Key] != nil && jsonToDbTag[cf.Key].Constraint == "" { return 0, obj, fmt.Errorf("constraint not defined for field %s", cf.Key) } @@ -78,7 +83,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac return 0, obj, err } - dynamicType := createDynamicStruct(t, fieldToJsonTags, 1) + dynamicType := utils.CreateDynamicStruct(t, fieldToJsonTags, 1) dynamicInstance := reflect.New(dynamicType).Interface() @@ -95,37 +100,22 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac // Check if we have at least one object in the response if len(result.Obj) == 0 { - return 0, obj, ErrNoObjFound - } - - // Map the dynamic struct to the final type T - finalObject := reflect.New(t).Interface() - gid, err = mapDynamicToFinal(result.Obj[0], finalObject, false) - if err != nil { - return 0, obj, err - } - - if typedPtr, ok := finalObject.(*T); ok { - return gid, *typedPtr, nil - } - - if dirType, ok := finalObject.(T); ok { - return gid, dirType, nil + return 0, obj, utils.ErrNoObjFound } - return 0, obj, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) + return utils.ConvertDynamicToTyped[T](result.Obj[0], t) } func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryParams, withReverse bool) ([]uint64, []T, error) { var obj T t := reflect.TypeOf(obj) - fieldToJsonTags, _, jsonToReverseEdgeTags, err := getFieldTags(t) + fieldToJsonTags, _, jsonToReverseEdgeTags, err := utils.GetFieldTags(t) if err != nil { return nil, nil, err } - var filterQueryFunc QueryFunc = func() string { + var filterQueryFunc query_gen.QueryFunc = func() string { return "" } var paginationAndSorting string @@ -146,18 +136,18 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar readFromQuery := "" if withReverse { for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { - readFromQuery += fmt.Sprintf(reverseEdgeQuery, getPredicateName(t.Name(), jsonTag), reverseEdgeTag) + readFromQuery += fmt.Sprintf(query_gen.ReverseEdgeQuery, utils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) } } - query := formatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) + query := query_gen.FormatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) resp, err := n.queryWithLock(ctx, query) if err != nil { return nil, nil, err } - dynamicType := createDynamicStruct(t, fieldToJsonTags, 1) + dynamicType := utils.CreateDynamicStruct(t, fieldToJsonTags, 1) var result struct { Objs []any `json:"objs"` @@ -181,25 +171,30 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar return nil, nil, err } - var gids []uint64 - var objs []T - for _, obj := range result.Objs { - finalObject := reflect.New(t).Interface() - gid, err := mapDynamicToFinal(obj, finalObject, false) + gids := make([]uint64, len(result.Objs)) + objs := make([]T, len(result.Objs)) + for i, obj := range result.Objs { + gid, typedObj, err := utils.ConvertDynamicToTyped[T](obj, t) if err != nil { return nil, nil, err } - - if typedPtr, ok := finalObject.(*T); ok { - gids = append(gids, gid) - objs = append(objs, *typedPtr) - } else if dirType, ok := finalObject.(T); ok { - gids = append(gids, gid) - objs = append(objs, dirType) - } else { - return nil, nil, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) - } + gids[i] = gid + objs[i] = typedObj } return gids, objs, nil } + +func getExistingObject[T any](ctx context.Context, n *Namespace, gid uint64, cf *ConstrainedField, + object T) (uint64, error) { + var err error + if gid != 0 { + gid, _, err = getByGidWithObject[T](ctx, n, gid, object) + } else if cf != nil { + gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, object) + } + if err != nil { + return 0, err + } + return gid, nil +} diff --git a/api_reflect.go b/api_reflect.go index 5a32b01..029a223 100644 --- a/api_reflect.go +++ b/api_reflect.go @@ -12,203 +12,17 @@ package modusdb import ( "fmt" "reflect" - "strconv" - "strings" -) - -type dbTag struct { - constraint string -} - -func getFieldTags(t reflect.Type) (fieldToJsonTags map[string]string, - jsonToDbTags map[string]*dbTag, jsonToReverseEdgeTags map[string]string, err error) { - - fieldToJsonTags = make(map[string]string) - jsonToDbTags = make(map[string]*dbTag) - jsonToReverseEdgeTags = make(map[string]string) - for i := 0; i < t.NumField(); i++ { - field := t.Field(i) - jsonTag := field.Tag.Get("json") - if jsonTag == "" { - return nil, nil, nil, fmt.Errorf("field %s has no json tag", field.Name) - } - jsonName := strings.Split(jsonTag, ",")[0] - fieldToJsonTags[field.Name] = jsonName - - reverseEdgeTag := field.Tag.Get("readFrom") - if reverseEdgeTag != "" { - typeAndField := strings.Split(reverseEdgeTag, ",") - if len(typeAndField) != 2 { - return nil, nil, nil, fmt.Errorf(`field %s has invalid readFrom tag, - expected format is type=,field=`, field.Name) - } - t := strings.Split(typeAndField[0], "=")[1] - f := strings.Split(typeAndField[1], "=")[1] - jsonToReverseEdgeTags[jsonName] = getPredicateName(t, f) - } - - dbConstraintsTag := field.Tag.Get("db") - if dbConstraintsTag != "" { - jsonToDbTags[jsonName] = &dbTag{} - dbTagsSplit := strings.Split(dbConstraintsTag, ",") - for _, dbTag := range dbTagsSplit { - split := strings.Split(dbTag, "=") - if split[0] == "constraint" { - jsonToDbTags[jsonName].constraint = split[1] - } - } - } - } - return fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, nil -} - -func getJsonTagToValues(object any, fieldToJsonTags map[string]string) map[string]any { - values := make(map[string]any) - v := reflect.ValueOf(object) - for v.Kind() == reflect.Ptr { - v = v.Elem() - } - for fieldName, jsonName := range fieldToJsonTags { - fieldValue := v.FieldByName(fieldName) - values[jsonName] = fieldValue.Interface() - } - return values -} - -func createDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, depth int) reflect.Type { - fields := make([]reflect.StructField, 0, len(fieldToJsonTags)) - for fieldName, jsonName := range fieldToJsonTags { - field, _ := t.FieldByName(fieldName) - if fieldName != "Gid" { - if field.Type.Kind() == reflect.Struct { - if depth <= 1 { - nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type) - nestedType := createDynamicStruct(field.Type, nestedFieldToJsonTags, depth+1) - fields = append(fields, reflect.StructField{ - Name: field.Name, - Type: nestedType, - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), - }) - } - } else if field.Type.Kind() == reflect.Ptr && - field.Type.Elem().Kind() == reflect.Struct { - nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type.Elem()) - nestedType := createDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) - fields = append(fields, reflect.StructField{ - Name: field.Name, - Type: reflect.PointerTo(nestedType), - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), - }) - } else if field.Type.Kind() == reflect.Slice && - field.Type.Elem().Kind() == reflect.Struct { - nestedFieldToJsonTags, _, _, _ := getFieldTags(field.Type.Elem()) - nestedType := createDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) - fields = append(fields, reflect.StructField{ - Name: field.Name, - Type: reflect.SliceOf(nestedType), - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), - }) - } else { - fields = append(fields, reflect.StructField{ - Name: field.Name, - Type: field.Type, - Tag: reflect.StructTag(fmt.Sprintf(`json:"%s.%s"`, t.Name(), jsonName)), - }) - } - - } - } - fields = append(fields, reflect.StructField{ - Name: "Gid", - Type: reflect.TypeOf(""), - Tag: reflect.StructTag(`json:"gid"`), - }, reflect.StructField{ - Name: "DgraphType", - Type: reflect.TypeOf([]string{}), - Tag: reflect.StructTag(`json:"dgraph.type"`), - }) - return reflect.StructOf(fields) -} - -func mapDynamicToFinal(dynamic any, final any, isNested bool) (uint64, error) { - vFinal := reflect.ValueOf(final).Elem() - vDynamic := reflect.ValueOf(dynamic).Elem() - - gid := uint64(0) - - for i := 0; i < vDynamic.NumField(); i++ { - - dynamicField := vDynamic.Type().Field(i) - dynamicFieldType := dynamicField.Type - dynamicValue := vDynamic.Field(i) - - var finalField reflect.Value - if dynamicField.Name == "Gid" { - finalField = vFinal.FieldByName("Gid") - gidStr := dynamicValue.String() - gid, _ = strconv.ParseUint(gidStr, 0, 64) - } else if dynamicField.Name == "DgraphType" { - fieldArrInterface := dynamicValue.Interface() - fieldArr, ok := fieldArrInterface.([]string) - if ok { - if len(fieldArr) == 0 { - if !isNested { - return 0, ErrNoObjFound - } else { - continue - } - } - } else { - return 0, fmt.Errorf("DgraphType field should be an array of strings") - } - } else { - finalField = vFinal.FieldByName(dynamicField.Name) - } - if dynamicFieldType.Kind() == reflect.Struct { - _, err := mapDynamicToFinal(dynamicValue.Addr().Interface(), finalField.Addr().Interface(), true) - if err != nil { - return 0, err - } - } else if dynamicFieldType.Kind() == reflect.Ptr && - dynamicFieldType.Elem().Kind() == reflect.Struct { - // if field is a pointer, find if the underlying is a struct - _, err := mapDynamicToFinal(dynamicValue.Interface(), finalField.Interface(), true) - if err != nil { - return 0, err - } - } else if dynamicFieldType.Kind() == reflect.Slice && - dynamicFieldType.Elem().Kind() == reflect.Struct { - for j := 0; j < dynamicValue.Len(); j++ { - sliceElem := dynamicValue.Index(j).Addr().Interface() - finalSliceElem := reflect.New(finalField.Type().Elem()).Elem() - _, err := mapDynamicToFinal(sliceElem, finalSliceElem.Addr().Interface(), true) - if err != nil { - return 0, err - } - finalField.Set(reflect.Append(finalField, finalSliceElem)) - } - } else { - if finalField.IsValid() && finalField.CanSet() { - // if field name is gid, convert it to uint64 - if dynamicField.Name == "Gid" { - finalField.SetUint(gid) - } else { - finalField.Set(dynamicValue) - } - } - } - } - return gid, nil -} + "github.com/hypermodeinc/modusdb/api/utils" +) -func getUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { +func GetUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { t := reflect.TypeOf(object) - fieldToJsonTags, jsonToDbTags, _, err := getFieldTags(t) + fieldToJsonTags, jsonToDbTags, _, err := utils.GetFieldTags(t) if err != nil { return 0, nil, err } - jsonTagToValue := getJsonTagToValues(object, fieldToJsonTags) + jsonTagToValue := utils.GetJsonTagToValues(object, fieldToJsonTags) for jsonName, value := range jsonTagToValue { if jsonName == "gid" { @@ -220,7 +34,7 @@ func getUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { return gid, nil, nil } } - if jsonToDbTags[jsonName] != nil && isValidUniqueIndex(jsonToDbTags[jsonName].constraint) { + if jsonToDbTags[jsonName] != nil && IsValidUniqueIndex(jsonToDbTags[jsonName].Constraint) { // check if value is zero or nil if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { continue @@ -232,9 +46,9 @@ func getUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { } } - return 0, nil, fmt.Errorf(NoUniqueConstr, t.Name()) + return 0, nil, fmt.Errorf(utils.NoUniqueConstr, t.Name()) } -func isValidUniqueIndex(name string) bool { +func IsValidUniqueIndex(name string) bool { return name == "unique" } diff --git a/api_test.go b/api_test.go index f7f15e0..cb0ca7a 100644 --- a/api_test.go +++ b/api_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/hypermodeinc/modusdb" + "github.com/hypermodeinc/modusdb/api/utils" ) type User struct { @@ -767,7 +768,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { _, _, err = modusdb.Create(db, branch, db1.ID()) require.Error(t, err) - require.Equal(t, fmt.Sprintf(modusdb.NoUniqueConstr, "BadProject"), err.Error()) + require.Equal(t, fmt.Sprintf(utils.NoUniqueConstr, "BadProject"), err.Error()) proj := BadProject{ Name: "P", @@ -776,7 +777,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { _, _, err = modusdb.Create(db, proj, db1.ID()) require.Error(t, err) - require.Equal(t, fmt.Sprintf(modusdb.NoUniqueConstr, "BadProject"), err.Error()) + require.Equal(t, fmt.Sprintf(utils.NoUniqueConstr, "BadProject"), err.Error()) } diff --git a/api_types.go b/api_types.go index 2384f35..90d0af1 100644 --- a/api_types.go +++ b/api_types.go @@ -11,22 +11,12 @@ package modusdb import ( "context" - "encoding/binary" "fmt" "strings" - "time" - "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/dgraph-io/dgraph/v24/protos/pb" - "github.com/dgraph-io/dgraph/v24/types" "github.com/dgraph-io/dgraph/v24/x" - "github.com/twpayne/go-geom" - "github.com/twpayne/go-geom/encoding/wkb" -) - -var ( - ErrNoObjFound = fmt.Errorf("no object found") - NoUniqueConstr = "unique constraint not defined for any field on type %s" + "github.com/hypermodeinc/modusdb/api/query_gen" + "github.com/hypermodeinc/modusdb/api/utils" ) type UniqueField interface { @@ -113,137 +103,108 @@ func getDefaultNamespace(db *DB, ns ...uint64) (context.Context, *Namespace, err return ctx, n, nil } -func valueToPosting_ValType(v any) (pb.Posting_ValType, error) { - switch v.(type) { - case string: - return pb.Posting_STRING, nil - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32: - return pb.Posting_INT, nil - case uint64: - return pb.Posting_UID, nil - case bool: - return pb.Posting_BOOL, nil - case float32, float64: - return pb.Posting_FLOAT, nil - case []byte: - return pb.Posting_BINARY, nil - case time.Time: - return pb.Posting_DATETIME, nil - case geom.Point: - return pb.Posting_GEO, nil - case []float32, []float64: - return pb.Posting_VFLOAT, nil - default: - return pb.Posting_DEFAULT, fmt.Errorf("unsupported type %T", v) - } -} - -func valueToApiVal(v any) (*api.Value, error) { - switch val := v.(type) { - case string: - return &api.Value{Val: &api.Value_StrVal{StrVal: val}}, nil - case int: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int8: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int16: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int32: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case int64: - return &api.Value{Val: &api.Value_IntVal{IntVal: val}}, nil - case uint8: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint16: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint32: - return &api.Value{Val: &api.Value_IntVal{IntVal: int64(val)}}, nil - case uint64: - return &api.Value{Val: &api.Value_UidVal{UidVal: val}}, nil - case bool: - return &api.Value{Val: &api.Value_BoolVal{BoolVal: val}}, nil - case float32: - return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: float64(val)}}, nil - case float64: - return &api.Value{Val: &api.Value_DoubleVal{DoubleVal: val}}, nil - case []float32: - return &api.Value{Val: &api.Value_Vfloat32Val{ - Vfloat32Val: types.FloatArrayAsBytes(val)}}, nil - case []float64: - float32Slice := make([]float32, len(val)) - for i, v := range val { - float32Slice[i] = float32(v) - } - return &api.Value{Val: &api.Value_Vfloat32Val{ - Vfloat32Val: types.FloatArrayAsBytes(float32Slice)}}, nil - case []byte: - return &api.Value{Val: &api.Value_BytesVal{BytesVal: val}}, nil - case time.Time: - bytes, err := val.MarshalBinary() - if err != nil { - return nil, err - } - return &api.Value{Val: &api.Value_DateVal{DateVal: bytes}}, nil - case geom.Point: - bytes, err := wkb.Marshal(&val, binary.LittleEndian) - if err != nil { - return nil, err - } - return &api.Value{Val: &api.Value_GeoVal{GeoVal: bytes}}, nil - case uint: - return &api.Value{Val: &api.Value_DefaultVal{DefaultVal: fmt.Sprint(v)}}, nil - default: - return nil, fmt.Errorf("unsupported type %T", v) - } -} - -func filterToQueryFunc(typeName string, f Filter) QueryFunc { +func filterToQueryFunc(typeName string, f Filter) query_gen.QueryFunc { // Handle logical operators first if f.And != nil { - return And(filterToQueryFunc(typeName, *f.And)) + return query_gen.And(filterToQueryFunc(typeName, *f.And)) } if f.Or != nil { - return Or(filterToQueryFunc(typeName, *f.Or)) + return query_gen.Or(filterToQueryFunc(typeName, *f.Or)) } if f.Not != nil { - return Not(filterToQueryFunc(typeName, *f.Not)) + return query_gen.Not(filterToQueryFunc(typeName, *f.Not)) } // Handle field predicates if f.String.Equals != "" { - return buildEqQuery(getPredicateName(typeName, f.Field), f.String.Equals) + return query_gen.BuildEqQuery(utils.GetPredicateName(typeName, f.Field), f.String.Equals) } if len(f.String.AllOfTerms) != 0 { - return buildAllOfTermsQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AllOfTerms, " ")) + return query_gen.BuildAllOfTermsQuery(utils.GetPredicateName(typeName, + f.Field), strings.Join(f.String.AllOfTerms, " ")) } if len(f.String.AnyOfTerms) != 0 { - return buildAnyOfTermsQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AnyOfTerms, " ")) + return query_gen.BuildAnyOfTermsQuery(utils.GetPredicateName(typeName, + f.Field), strings.Join(f.String.AnyOfTerms, " ")) } if len(f.String.AllOfText) != 0 { - return buildAllOfTextQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AllOfText, " ")) + return query_gen.BuildAllOfTextQuery(utils.GetPredicateName(typeName, + f.Field), strings.Join(f.String.AllOfText, " ")) } if len(f.String.AnyOfText) != 0 { - return buildAnyOfTextQuery(getPredicateName(typeName, f.Field), strings.Join(f.String.AnyOfText, " ")) + return query_gen.BuildAnyOfTextQuery(utils.GetPredicateName(typeName, + f.Field), strings.Join(f.String.AnyOfText, " ")) } if f.String.RegExp != "" { - return buildRegExpQuery(getPredicateName(typeName, f.Field), f.String.RegExp) + return query_gen.BuildRegExpQuery(utils.GetPredicateName(typeName, + f.Field), f.String.RegExp) } if f.String.LessThan != "" { - return buildLtQuery(getPredicateName(typeName, f.Field), f.String.LessThan) + return query_gen.BuildLtQuery(utils.GetPredicateName(typeName, + f.Field), f.String.LessThan) } if f.String.LessOrEqual != "" { - return buildLeQuery(getPredicateName(typeName, f.Field), f.String.LessOrEqual) + return query_gen.BuildLeQuery(utils.GetPredicateName(typeName, + f.Field), f.String.LessOrEqual) } if f.String.GreaterThan != "" { - return buildGtQuery(getPredicateName(typeName, f.Field), f.String.GreaterThan) + return query_gen.BuildGtQuery(utils.GetPredicateName(typeName, + f.Field), f.String.GreaterThan) } if f.String.GreaterOrEqual != "" { - return buildGeQuery(getPredicateName(typeName, f.Field), f.String.GreaterOrEqual) + return query_gen.BuildGeQuery(utils.GetPredicateName(typeName, + f.Field), f.String.GreaterOrEqual) } if f.Vector.SimilarTo != nil { - return buildSimilarToQuery(getPredicateName(typeName, f.Field), f.Vector.TopK, f.Vector.SimilarTo) + return query_gen.BuildSimilarToQuery(utils.GetPredicateName(typeName, + f.Field), f.Vector.TopK, f.Vector.SimilarTo) } // Return empty query if no conditions match return func() string { return "" } } + +// Helper function to combine multiple filters +func filtersToQueryFunc(typeName string, filter Filter) query_gen.QueryFunc { + return filterToQueryFunc(typeName, filter) +} + +func paginationToQueryString(p Pagination) string { + paginationStr := "" + if p.Limit > 0 { + paginationStr += ", " + fmt.Sprintf("first: %d", p.Limit) + } + if p.Offset > 0 { + paginationStr += ", " + fmt.Sprintf("offset: %d", p.Offset) + } else if p.After != "" { + paginationStr += ", " + fmt.Sprintf("after: %s", p.After) + } + if paginationStr == "" { + return "" + } + return paginationStr +} + +func sortingToQueryString(typeName string, s Sorting) string { + if s.OrderAscField == "" && s.OrderDescField == "" { + return "" + } + + var parts []string + first, second := s.OrderDescField, s.OrderAscField + firstOp, secondOp := "orderdesc", "orderasc" + + if !s.OrderDescFirst { + first, second = s.OrderAscField, s.OrderDescField + firstOp, secondOp = "orderasc", "orderdesc" + } + + if first != "" { + parts = append(parts, fmt.Sprintf("%s: %s", firstOp, utils.GetPredicateName(typeName, first))) + } + if second != "" { + parts = append(parts, fmt.Sprintf("%s: %s", secondOp, utils.GetPredicateName(typeName, second))) + } + + return ", " + strings.Join(parts, ", ") +} From 7abdc7244d9be76d2e4422ccce407d1f0759b539 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Sat, 4 Jan 2025 16:12:12 -0800 Subject: [PATCH 20/23] chore: move reflection logic to utils package (#54) **Description** more refactoring to simplify code in main modusdb package **Checklist** - [x] Code compiles correctly and linting passes locally --- .trunk/trunk.yaml | 2 +- api.go | 9 ++++++- api/utils/reflect.go | 34 ++++++++++++++++++++++++++ api/utils/utils.go | 13 ++++++++++ api_mutation_gen.go | 4 +-- api_mutation_helpers.go | 6 ++++- api_reflect.go | 54 ----------------------------------------- 7 files changed, 63 insertions(+), 59 deletions(-) delete mode 100644 api_reflect.go diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 85fae60..9442ef2 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -31,7 +31,7 @@ lint: - markdownlint@0.43.0 - osv-scanner@1.9.2 - prettier@3.4.2 - - renovate@39.88.0 + - renovate@39.90.2 - trufflehog@3.88.0 - yamllint@1.35.1 diff --git a/api.go b/api.go index beadf9f..ff10a00 100644 --- a/api.go +++ b/api.go @@ -67,10 +67,17 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { return 0, object, false, err } - gid, cf, err := GetUniqueConstraint[T](object) + gid, cfKeyValue, err := utils.GetUniqueConstraint[T](object) if err != nil { return 0, object, false, err } + var cf *ConstrainedField + if cfKeyValue != nil { + cf = &ConstrainedField{ + Key: cfKeyValue.Key(), + Value: cfKeyValue.Value(), + } + } dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} diff --git a/api/utils/reflect.go b/api/utils/reflect.go index a4a57e0..fb2cf04 100644 --- a/api/utils/reflect.go +++ b/api/utils/reflect.go @@ -208,3 +208,37 @@ func ConvertDynamicToTyped[T any](obj any, t reflect.Type) (uint64, T, error) { } return 0, result, fmt.Errorf("failed to convert type %T to %T", finalObject, obj) } + +func GetUniqueConstraint[T any](object T) (uint64, *keyValue, error) { + t := reflect.TypeOf(object) + fieldToJsonTags, jsonToDbTags, _, err := GetFieldTags(t) + if err != nil { + return 0, nil, err + } + jsonTagToValue := GetJsonTagToValues(object, fieldToJsonTags) + + for jsonName, value := range jsonTagToValue { + if jsonName == "gid" { + gid, ok := value.(uint64) + if !ok { + continue + } + if gid != 0 { + return gid, nil, nil + } + } + if jsonToDbTags[jsonName] != nil && IsValidUniqueIndex(jsonToDbTags[jsonName].Constraint) { + // check if value is zero or nil + if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { + continue + } + return 0, &keyValue{key: jsonName, value: value}, nil + } + } + + return 0, nil, fmt.Errorf(NoUniqueConstr, t.Name()) +} + +func IsValidUniqueIndex(name string) bool { + return name == "unique" +} diff --git a/api/utils/utils.go b/api/utils/utils.go index ba726ed..03c2215 100644 --- a/api/utils/utils.go +++ b/api/utils/utils.go @@ -20,6 +20,19 @@ var ( NoUniqueConstr = "unique constraint not defined for any field on type %s" ) +type keyValue struct { + key string + value any +} + +func (kv *keyValue) Key() string { + return kv.key +} + +func (kv *keyValue) Value() any { + return kv.value +} + func GetPredicateName(typeName, fieldName string) string { return fmt.Sprint(typeName, ".", fieldName) } diff --git a/api_mutation_gen.go b/api_mutation_gen.go index 3bfdc6f..fb3b6c8 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -44,7 +44,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, var nquad *api.NQuad if jsonToReverseEdgeTags[jsonName] != "" { - if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.id, sch, jsonToReverseEdgeTags); err != nil { + if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, jsonToReverseEdgeTags); err != nil { return err } continue @@ -82,7 +82,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, } sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: utils.AddNamespace(n.id, t.Name()), + TypeName: utils.AddNamespace(n.ID(), t.Name()), Fields: sch.Preds, }) diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go index d4b0cf4..2016f04 100644 --- a/api_mutation_helpers.go +++ b/api_mutation_helpers.go @@ -38,10 +38,14 @@ func processPointerValue(ctx context.Context, value any, n *Namespace) (any, err } func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) (uint64, error) { - gid, cf, err := GetUniqueConstraint[T](object) + gid, cfKeyValue, err := utils.GetUniqueConstraint[T](object) if err != nil { return 0, err } + var cf *ConstrainedField + if cfKeyValue != nil { + cf = &ConstrainedField{Key: cfKeyValue.Key(), Value: cfKeyValue.Value()} + } dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} diff --git a/api_reflect.go b/api_reflect.go deleted file mode 100644 index 029a223..0000000 --- a/api_reflect.go +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2025 Hypermode Inc. - * Licensed under the terms of the Apache License, Version 2.0 - * See the LICENSE file that accompanied this code for further details. - * - * SPDX-FileCopyrightText: 2025 Hypermode Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package modusdb - -import ( - "fmt" - "reflect" - - "github.com/hypermodeinc/modusdb/api/utils" -) - -func GetUniqueConstraint[T any](object T) (uint64, *ConstrainedField, error) { - t := reflect.TypeOf(object) - fieldToJsonTags, jsonToDbTags, _, err := utils.GetFieldTags(t) - if err != nil { - return 0, nil, err - } - jsonTagToValue := utils.GetJsonTagToValues(object, fieldToJsonTags) - - for jsonName, value := range jsonTagToValue { - if jsonName == "gid" { - gid, ok := value.(uint64) - if !ok { - continue - } - if gid != 0 { - return gid, nil, nil - } - } - if jsonToDbTags[jsonName] != nil && IsValidUniqueIndex(jsonToDbTags[jsonName].Constraint) { - // check if value is zero or nil - if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { - continue - } - return 0, &ConstrainedField{ - Key: jsonName, - Value: value, - }, nil - } - } - - return 0, nil, fmt.Errorf(utils.NoUniqueConstr, t.Name()) -} - -func IsValidUniqueIndex(name string) bool { - return name == "unique" -} From 48b27dee2a8c026f7d293c5555c8e13e7c33377c Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Mon, 6 Jan 2025 00:19:49 -0800 Subject: [PATCH 21/23] revs --- api.go | 7 +- api_mutation_gen.go | 22 ++-- api_mutation_helpers.go | 7 +- api_query_execution.go | 39 ++++--- api_test.go | 6 +- api_types.go | 40 +++---- .../utils.go => internal/apiutils/apiutils.go | 15 +-- .../dgraphtypes/dgraphtypes.go | 14 ++- {api => internal}/mutations/mutations.go | 24 ++-- .../querygen}/dql_query.go | 11 +- internal/structreflect/keyval.go | 23 ++++ .../structreflect/structreflect.go | 110 +++++++----------- internal/structreflect/tagparser.go | 59 ++++++++++ internal/structreflect/tags.go | 20 ++++ internal/structreflect/value_extractor.go | 28 +++++ 15 files changed, 278 insertions(+), 147 deletions(-) rename api/utils/utils.go => internal/apiutils/apiutils.go (79%) rename api/utils/dgraph.go => internal/dgraphtypes/dgraphtypes.go (91%) rename {api => internal}/mutations/mutations.go (65%) rename {api/query_gen => internal/querygen}/dql_query.go (92%) create mode 100644 internal/structreflect/keyval.go rename api/utils/reflect.go => internal/structreflect/structreflect.go (63%) create mode 100644 internal/structreflect/tagparser.go create mode 100644 internal/structreflect/tags.go create mode 100644 internal/structreflect/value_extractor.go diff --git a/api.go b/api.go index ff10a00..0b06df5 100644 --- a/api.go +++ b/api.go @@ -14,7 +14,8 @@ import ( "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/schema" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/internal/structreflect" ) func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { @@ -67,7 +68,7 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { return 0, object, false, err } - gid, cfKeyValue, err := utils.GetUniqueConstraint[T](object) + gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) if err != nil { return 0, object, false, err } @@ -93,7 +94,7 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { if gid != 0 || cf != nil { gid, err = getExistingObject[T](ctx, n, gid, cf, object) - if err != nil && err != utils.ErrNoObjFound { + if err != nil && err != apiutils.ErrNoObjFound { return 0, object, false, err } wasFound = err == nil diff --git a/api_mutation_gen.go b/api_mutation_gen.go index fb3b6c8..5c3470b 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -19,8 +19,10 @@ import ( "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/x" - "github.com/hypermodeinc/modusdb/api/mutations" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/internal/dgraphtypes" + "github.com/hypermodeinc/modusdb/internal/mutations" + "github.com/hypermodeinc/modusdb/internal/structreflect" ) func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, @@ -30,11 +32,11 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, return fmt.Errorf("expected struct, got %s", t.Kind()) } - fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, err := utils.GetFieldTags(t) + tagMaps, err := structreflect.GetFieldTags(t) if err != nil { return err } - jsonTagToValue := utils.GetJsonTagToValues(object, fieldToJsonTags) + jsonTagToValue := structreflect.GetJsonTagToValues(object, tagMaps.FieldToJson) nquads := make([]*api.NQuad, 0) uniqueConstraintFound := false @@ -43,8 +45,8 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, reflectValueType := reflect.TypeOf(value) var nquad *api.NQuad - if jsonToReverseEdgeTags[jsonName] != "" { - if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, jsonToReverseEdgeTags); err != nil { + if tagMaps.JsonToReverseEdge[jsonName] != "" { + if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, tagMaps.JsonToReverseEdge); err != nil { return err } continue @@ -69,7 +71,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, return err } - uniqueConstraintFound, err = utils.HandleConstraints(u, jsonToDbTags, jsonName, u.ValueType, uniqueConstraintFound) + uniqueConstraintFound, err = dgraphtypes.HandleConstraints(u, tagMaps.JsonToDb, jsonName, u.ValueType, uniqueConstraintFound) if err != nil { return err } @@ -78,15 +80,15 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, nquads = append(nquads, nquad) } if !uniqueConstraintFound { - return fmt.Errorf(utils.NoUniqueConstr, t.Name()) + return fmt.Errorf(apiutils.NoUniqueConstr, t.Name()) } sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: utils.AddNamespace(n.ID(), t.Name()), + TypeName: apiutils.AddNamespace(n.ID(), t.Name()), Fields: sch.Preds, }) - val, err := utils.ValueToApiVal(t.Name()) + val, err := dgraphtypes.ValueToApiVal(t.Name()) if err != nil { return err } diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go index 2016f04..aba02bc 100644 --- a/api_mutation_helpers.go +++ b/api_mutation_helpers.go @@ -10,7 +10,8 @@ import ( "github.com/dgraph-io/dgraph/v24/query" "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/worker" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/internal/structreflect" ) func processStructValue(ctx context.Context, value any, n *Namespace) (any, error) { @@ -38,7 +39,7 @@ func processPointerValue(ctx context.Context, value any, n *Namespace) (any, err } func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) (uint64, error) { - gid, cfKeyValue, err := utils.GetUniqueConstraint[T](object) + gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) if err != nil { return 0, err } @@ -60,7 +61,7 @@ func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) } if gid != 0 || cf != nil { gid, err = getExistingObject(ctx, n, gid, cf, object) - if err != nil && err != utils.ErrNoObjFound { + if err != nil && err != apiutils.ErrNoObjFound { return 0, err } if err == nil { diff --git a/api_query_execution.go b/api_query_execution.go index bf9cb23..ec71bc2 100644 --- a/api_query_execution.go +++ b/api_query_execution.go @@ -15,8 +15,9 @@ import ( "fmt" "reflect" - "github.com/hypermodeinc/modusdb/api/query_gen" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/internal/querygen" + "github.com/hypermodeinc/modusdb/internal/structreflect" ) func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, T, error) { @@ -50,15 +51,15 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac obj T, withReverse bool, args ...R) (uint64, T, error) { t := reflect.TypeOf(obj) - fieldToJsonTags, jsonToDbTag, jsonToReverseEdgeTags, err := utils.GetFieldTags(t) + tagMaps, err := structreflect.GetFieldTags(t) if err != nil { return 0, obj, err } readFromQuery := "" if withReverse { - for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { - readFromQuery += fmt.Sprintf(query_gen.ReverseEdgeQuery, - utils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) + for jsonTag, reverseEdgeTag := range tagMaps.JsonToReverseEdge { + readFromQuery += fmt.Sprintf(querygen.ReverseEdgeQuery, + apiutils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) } } @@ -66,15 +67,15 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac var query string gid, ok := any(args[0]).(uint64) if ok { - query = query_gen.FormatObjQuery(query_gen.BuildUidQuery(gid), readFromQuery) + query = querygen.FormatObjQuery(querygen.BuildUidQuery(gid), readFromQuery) } else if cf, ok = any(args[0]).(ConstrainedField); ok { - query = query_gen.FormatObjQuery(query_gen.BuildEqQuery(utils.GetPredicateName(t.Name(), + query = querygen.FormatObjQuery(querygen.BuildEqQuery(apiutils.GetPredicateName(t.Name(), cf.Key), cf.Value), readFromQuery) } else { return 0, obj, fmt.Errorf("invalid unique field type") } - if jsonToDbTag[cf.Key] != nil && jsonToDbTag[cf.Key].Constraint == "" { + if tagMaps.JsonToDb[cf.Key] != nil && tagMaps.JsonToDb[cf.Key].Constraint == "" { return 0, obj, fmt.Errorf("constraint not defined for field %s", cf.Key) } @@ -83,7 +84,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac return 0, obj, err } - dynamicType := utils.CreateDynamicStruct(t, fieldToJsonTags, 1) + dynamicType := structreflect.CreateDynamicStruct(t, tagMaps.FieldToJson, 1) dynamicInstance := reflect.New(dynamicType).Interface() @@ -100,22 +101,22 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac // Check if we have at least one object in the response if len(result.Obj) == 0 { - return 0, obj, utils.ErrNoObjFound + return 0, obj, apiutils.ErrNoObjFound } - return utils.ConvertDynamicToTyped[T](result.Obj[0], t) + return structreflect.ConvertDynamicToTyped[T](result.Obj[0], t) } func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryParams, withReverse bool) ([]uint64, []T, error) { var obj T t := reflect.TypeOf(obj) - fieldToJsonTags, _, jsonToReverseEdgeTags, err := utils.GetFieldTags(t) + tagMaps, err := structreflect.GetFieldTags(t) if err != nil { return nil, nil, err } - var filterQueryFunc query_gen.QueryFunc = func() string { + var filterQueryFunc querygen.QueryFunc = func() string { return "" } var paginationAndSorting string @@ -135,19 +136,19 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar readFromQuery := "" if withReverse { - for jsonTag, reverseEdgeTag := range jsonToReverseEdgeTags { - readFromQuery += fmt.Sprintf(query_gen.ReverseEdgeQuery, utils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) + for jsonTag, reverseEdgeTag := range tagMaps.JsonToReverseEdge { + readFromQuery += fmt.Sprintf(querygen.ReverseEdgeQuery, apiutils.GetPredicateName(t.Name(), jsonTag), reverseEdgeTag) } } - query := query_gen.FormatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) + query := querygen.FormatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) resp, err := n.queryWithLock(ctx, query) if err != nil { return nil, nil, err } - dynamicType := utils.CreateDynamicStruct(t, fieldToJsonTags, 1) + dynamicType := structreflect.CreateDynamicStruct(t, tagMaps.FieldToJson, 1) var result struct { Objs []any `json:"objs"` @@ -174,7 +175,7 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar gids := make([]uint64, len(result.Objs)) objs := make([]T, len(result.Objs)) for i, obj := range result.Objs { - gid, typedObj, err := utils.ConvertDynamicToTyped[T](obj, t) + gid, typedObj, err := structreflect.ConvertDynamicToTyped[T](obj, t) if err != nil { return nil, nil, err } diff --git a/api_test.go b/api_test.go index cb0ca7a..a08b461 100644 --- a/api_test.go +++ b/api_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/hypermodeinc/modusdb" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" ) type User struct { @@ -768,7 +768,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { _, _, err = modusdb.Create(db, branch, db1.ID()) require.Error(t, err) - require.Equal(t, fmt.Sprintf(utils.NoUniqueConstr, "BadProject"), err.Error()) + require.Equal(t, fmt.Sprintf(apiutils.NoUniqueConstr, "BadProject"), err.Error()) proj := BadProject{ Name: "P", @@ -777,7 +777,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { _, _, err = modusdb.Create(db, proj, db1.ID()) require.Error(t, err) - require.Equal(t, fmt.Sprintf(utils.NoUniqueConstr, "BadProject"), err.Error()) + require.Equal(t, fmt.Sprintf(apiutils.NoUniqueConstr, "BadProject"), err.Error()) } diff --git a/api_types.go b/api_types.go index 90d0af1..c81651c 100644 --- a/api_types.go +++ b/api_types.go @@ -15,8 +15,8 @@ import ( "strings" "github.com/dgraph-io/dgraph/v24/x" - "github.com/hypermodeinc/modusdb/api/query_gen" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/internal/querygen" ) type UniqueField interface { @@ -103,60 +103,60 @@ func getDefaultNamespace(db *DB, ns ...uint64) (context.Context, *Namespace, err return ctx, n, nil } -func filterToQueryFunc(typeName string, f Filter) query_gen.QueryFunc { +func filterToQueryFunc(typeName string, f Filter) querygen.QueryFunc { // Handle logical operators first if f.And != nil { - return query_gen.And(filterToQueryFunc(typeName, *f.And)) + return querygen.And(filterToQueryFunc(typeName, *f.And)) } if f.Or != nil { - return query_gen.Or(filterToQueryFunc(typeName, *f.Or)) + return querygen.Or(filterToQueryFunc(typeName, *f.Or)) } if f.Not != nil { - return query_gen.Not(filterToQueryFunc(typeName, *f.Not)) + return querygen.Not(filterToQueryFunc(typeName, *f.Not)) } // Handle field predicates if f.String.Equals != "" { - return query_gen.BuildEqQuery(utils.GetPredicateName(typeName, f.Field), f.String.Equals) + return querygen.BuildEqQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.Equals) } if len(f.String.AllOfTerms) != 0 { - return query_gen.BuildAllOfTermsQuery(utils.GetPredicateName(typeName, + return querygen.BuildAllOfTermsQuery(apiutils.GetPredicateName(typeName, f.Field), strings.Join(f.String.AllOfTerms, " ")) } if len(f.String.AnyOfTerms) != 0 { - return query_gen.BuildAnyOfTermsQuery(utils.GetPredicateName(typeName, + return querygen.BuildAnyOfTermsQuery(apiutils.GetPredicateName(typeName, f.Field), strings.Join(f.String.AnyOfTerms, " ")) } if len(f.String.AllOfText) != 0 { - return query_gen.BuildAllOfTextQuery(utils.GetPredicateName(typeName, + return querygen.BuildAllOfTextQuery(apiutils.GetPredicateName(typeName, f.Field), strings.Join(f.String.AllOfText, " ")) } if len(f.String.AnyOfText) != 0 { - return query_gen.BuildAnyOfTextQuery(utils.GetPredicateName(typeName, + return querygen.BuildAnyOfTextQuery(apiutils.GetPredicateName(typeName, f.Field), strings.Join(f.String.AnyOfText, " ")) } if f.String.RegExp != "" { - return query_gen.BuildRegExpQuery(utils.GetPredicateName(typeName, + return querygen.BuildRegExpQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.RegExp) } if f.String.LessThan != "" { - return query_gen.BuildLtQuery(utils.GetPredicateName(typeName, + return querygen.BuildLtQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.LessThan) } if f.String.LessOrEqual != "" { - return query_gen.BuildLeQuery(utils.GetPredicateName(typeName, + return querygen.BuildLeQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.LessOrEqual) } if f.String.GreaterThan != "" { - return query_gen.BuildGtQuery(utils.GetPredicateName(typeName, + return querygen.BuildGtQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.GreaterThan) } if f.String.GreaterOrEqual != "" { - return query_gen.BuildGeQuery(utils.GetPredicateName(typeName, + return querygen.BuildGeQuery(apiutils.GetPredicateName(typeName, f.Field), f.String.GreaterOrEqual) } if f.Vector.SimilarTo != nil { - return query_gen.BuildSimilarToQuery(utils.GetPredicateName(typeName, + return querygen.BuildSimilarToQuery(apiutils.GetPredicateName(typeName, f.Field), f.Vector.TopK, f.Vector.SimilarTo) } @@ -165,7 +165,7 @@ func filterToQueryFunc(typeName string, f Filter) query_gen.QueryFunc { } // Helper function to combine multiple filters -func filtersToQueryFunc(typeName string, filter Filter) query_gen.QueryFunc { +func filtersToQueryFunc(typeName string, filter Filter) querygen.QueryFunc { return filterToQueryFunc(typeName, filter) } @@ -200,10 +200,10 @@ func sortingToQueryString(typeName string, s Sorting) string { } if first != "" { - parts = append(parts, fmt.Sprintf("%s: %s", firstOp, utils.GetPredicateName(typeName, first))) + parts = append(parts, fmt.Sprintf("%s: %s", firstOp, apiutils.GetPredicateName(typeName, first))) } if second != "" { - parts = append(parts, fmt.Sprintf("%s: %s", secondOp, utils.GetPredicateName(typeName, second))) + parts = append(parts, fmt.Sprintf("%s: %s", secondOp, apiutils.GetPredicateName(typeName, second))) } return ", " + strings.Join(parts, ", ") diff --git a/api/utils/utils.go b/internal/apiutils/apiutils.go similarity index 79% rename from api/utils/utils.go rename to internal/apiutils/apiutils.go index 03c2215..84d2696 100644 --- a/api/utils/utils.go +++ b/internal/apiutils/apiutils.go @@ -7,7 +7,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package utils +package apiutils import ( "fmt" @@ -20,19 +20,6 @@ var ( NoUniqueConstr = "unique constraint not defined for any field on type %s" ) -type keyValue struct { - key string - value any -} - -func (kv *keyValue) Key() string { - return kv.key -} - -func (kv *keyValue) Value() any { - return kv.value -} - func GetPredicateName(typeName, fieldName string) string { return fmt.Sprint(typeName, ".", fieldName) } diff --git a/api/utils/dgraph.go b/internal/dgraphtypes/dgraphtypes.go similarity index 91% rename from api/utils/dgraph.go rename to internal/dgraphtypes/dgraphtypes.go index fc49050..3e4e93a 100644 --- a/api/utils/dgraph.go +++ b/internal/dgraphtypes/dgraphtypes.go @@ -1,4 +1,13 @@ -package utils +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package dgraphtypes import ( "encoding/binary" @@ -8,6 +17,7 @@ import ( "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/types" + "github.com/hypermodeinc/modusdb/internal/structreflect" "github.com/twpayne/go-geom" "github.com/twpayne/go-geom/encoding/wkb" ) @@ -132,7 +142,7 @@ func ValueToApiVal(v any) (*api.Value, error) { } } -func HandleConstraints(u *pb.SchemaUpdate, jsonToDbTags map[string]*DbTag, jsonName string, +func HandleConstraints(u *pb.SchemaUpdate, jsonToDbTags map[string]*structreflect.DbTag, jsonName string, valType pb.Posting_ValType, uniqueConstraintFound bool) (bool, error) { if jsonToDbTags[jsonName] == nil { return uniqueConstraintFound, nil diff --git a/api/mutations/mutations.go b/internal/mutations/mutations.go similarity index 65% rename from api/mutations/mutations.go rename to internal/mutations/mutations.go index 4e4fedf..a8bdb7e 100644 --- a/api/mutations/mutations.go +++ b/internal/mutations/mutations.go @@ -1,3 +1,12 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + package mutations import ( @@ -8,7 +17,8 @@ import ( "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/schema" - "github.com/hypermodeinc/modusdb/api/utils" + "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/internal/dgraphtypes" ) func HandleReverseEdge(jsonName string, value reflect.Type, nsId uint64, sch *schema.ParsedSchema, @@ -24,14 +34,14 @@ func HandleReverseEdge(jsonName string, value reflect.Type, nsId uint64, sch *sc reverseEdge := jsonToReverseEdgeTags[jsonName] typeName := strings.Split(reverseEdge, ".")[0] u := &pb.SchemaUpdate{ - Predicate: utils.AddNamespace(nsId, reverseEdge), + Predicate: apiutils.AddNamespace(nsId, reverseEdge), ValueType: pb.Posting_UID, Directive: pb.SchemaUpdate_REVERSE, } sch.Preds = append(sch.Preds, u) sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: utils.AddNamespace(nsId, typeName), + TypeName: apiutils.AddNamespace(nsId, typeName), Fields: []*pb.SchemaUpdate{u}, }) return nil @@ -39,12 +49,12 @@ func HandleReverseEdge(jsonName string, value reflect.Type, nsId uint64, sch *sc func CreateNQuadAndSchema(value any, gid uint64, jsonName string, t reflect.Type, nsId uint64) (*api.NQuad, *pb.SchemaUpdate, error) { - valType, err := utils.ValueToPosting_ValType(value) + valType, err := dgraphtypes.ValueToPosting_ValType(value) if err != nil { return nil, nil, err } - val, err := utils.ValueToApiVal(value) + val, err := dgraphtypes.ValueToApiVal(value) if err != nil { return nil, nil, err } @@ -52,11 +62,11 @@ func CreateNQuadAndSchema(value any, gid uint64, jsonName string, t reflect.Type nquad := &api.NQuad{ Namespace: nsId, Subject: fmt.Sprint(gid), - Predicate: utils.GetPredicateName(t.Name(), jsonName), + Predicate: apiutils.GetPredicateName(t.Name(), jsonName), } u := &pb.SchemaUpdate{ - Predicate: utils.AddNamespace(nsId, utils.GetPredicateName(t.Name(), jsonName)), + Predicate: apiutils.AddNamespace(nsId, apiutils.GetPredicateName(t.Name(), jsonName)), ValueType: valType, } diff --git a/api/query_gen/dql_query.go b/internal/querygen/dql_query.go similarity index 92% rename from api/query_gen/dql_query.go rename to internal/querygen/dql_query.go index 5922342..f99234b 100644 --- a/api/query_gen/dql_query.go +++ b/internal/querygen/dql_query.go @@ -1,4 +1,13 @@ -package query_gen +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package querygen import ( "fmt" diff --git a/internal/structreflect/keyval.go b/internal/structreflect/keyval.go new file mode 100644 index 0000000..bfa5bdb --- /dev/null +++ b/internal/structreflect/keyval.go @@ -0,0 +1,23 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package structreflect + +type keyValue struct { + key string + value any +} + +func (kv *keyValue) Key() string { + return kv.key +} + +func (kv *keyValue) Value() any { + return kv.value +} diff --git a/api/utils/reflect.go b/internal/structreflect/structreflect.go similarity index 63% rename from api/utils/reflect.go rename to internal/structreflect/structreflect.go index fb2cf04..ec6b970 100644 --- a/api/utils/reflect.go +++ b/internal/structreflect/structreflect.go @@ -1,81 +1,61 @@ -package utils +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package structreflect import ( "fmt" "reflect" "strconv" - "strings" -) -type DbTag struct { - Constraint string -} + "github.com/hypermodeinc/modusdb/internal/apiutils" +) -func GetFieldTags(t reflect.Type) (fieldToJsonTags map[string]string, - jsonToDbTags map[string]*DbTag, jsonToReverseEdgeTags map[string]string, err error) { +func GetFieldTags(t reflect.Type) (*TagMaps, error) { + tags := &TagMaps{ + FieldToJson: make(map[string]string), + JsonToDb: make(map[string]*DbTag), + JsonToReverseEdge: make(map[string]string), + } - fieldToJsonTags = make(map[string]string) - jsonToDbTags = make(map[string]*DbTag) - jsonToReverseEdgeTags = make(map[string]string) for i := 0; i < t.NumField(); i++ { field := t.Field(i) - jsonTag := field.Tag.Get("json") - if jsonTag == "" { - return nil, nil, nil, fmt.Errorf("field %s has no json tag", field.Name) - } - jsonName := strings.Split(jsonTag, ",")[0] - fieldToJsonTags[field.Name] = jsonName - - reverseEdgeTag := field.Tag.Get("readFrom") - if reverseEdgeTag != "" { - typeAndField := strings.Split(reverseEdgeTag, ",") - if len(typeAndField) != 2 { - return nil, nil, nil, fmt.Errorf(`field %s has invalid readFrom tag, - expected format is type=,field=`, field.Name) - } - t := strings.Split(typeAndField[0], "=")[1] - f := strings.Split(typeAndField[1], "=")[1] - jsonToReverseEdgeTags[jsonName] = GetPredicateName(t, f) + + jsonName, err := parseJsonTag(field) + if err != nil { + return nil, err } + tags.FieldToJson[field.Name] = jsonName - dbConstraintsTag := field.Tag.Get("db") - if dbConstraintsTag != "" { - jsonToDbTags[jsonName] = &DbTag{} - dbTagsSplit := strings.Split(dbConstraintsTag, ",") - for _, dbTag := range dbTagsSplit { - split := strings.Split(dbTag, "=") - if split[0] == "constraint" { - jsonToDbTags[jsonName].Constraint = split[1] - } - } + if reverseEdge, err := parseReverseEdgeTag(field); err != nil { + return nil, err + } else if reverseEdge != "" { + tags.JsonToReverseEdge[jsonName] = reverseEdge } - } - return fieldToJsonTags, jsonToDbTags, jsonToReverseEdgeTags, nil -} -func GetJsonTagToValues(object any, fieldToJsonTags map[string]string) map[string]any { - values := make(map[string]any) - v := reflect.ValueOf(object) - for v.Kind() == reflect.Ptr { - v = v.Elem() + if dbTag := parseDbTag(field); dbTag != nil { + tags.JsonToDb[jsonName] = dbTag + } } - for fieldName, jsonName := range fieldToJsonTags { - fieldValue := v.FieldByName(fieldName) - values[jsonName] = fieldValue.Interface() - } - return values + return tags, nil } -func CreateDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, depth int) reflect.Type { - fields := make([]reflect.StructField, 0, len(fieldToJsonTags)) - for fieldName, jsonName := range fieldToJsonTags { +func CreateDynamicStruct(t reflect.Type, fieldToJson map[string]string, depth int) reflect.Type { + fields := make([]reflect.StructField, 0, len(fieldToJson)) + for fieldName, jsonName := range fieldToJson { field, _ := t.FieldByName(fieldName) if fieldName != "Gid" { if field.Type.Kind() == reflect.Struct { if depth <= 1 { - nestedFieldToJsonTags, _, _, _ := GetFieldTags(field.Type) - nestedType := CreateDynamicStruct(field.Type, nestedFieldToJsonTags, depth+1) + tagMaps, _ := GetFieldTags(field.Type) + nestedType := CreateDynamicStruct(field.Type, tagMaps.FieldToJson, depth+1) fields = append(fields, reflect.StructField{ Name: field.Name, Type: nestedType, @@ -84,8 +64,8 @@ func CreateDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, dept } } else if field.Type.Kind() == reflect.Ptr && field.Type.Elem().Kind() == reflect.Struct { - nestedFieldToJsonTags, _, _, _ := GetFieldTags(field.Type.Elem()) - nestedType := CreateDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) + tagMaps, _ := GetFieldTags(field.Type.Elem()) + nestedType := CreateDynamicStruct(field.Type.Elem(), tagMaps.FieldToJson, depth+1) fields = append(fields, reflect.StructField{ Name: field.Name, Type: reflect.PointerTo(nestedType), @@ -93,8 +73,8 @@ func CreateDynamicStruct(t reflect.Type, fieldToJsonTags map[string]string, dept }) } else if field.Type.Kind() == reflect.Slice && field.Type.Elem().Kind() == reflect.Struct { - nestedFieldToJsonTags, _, _, _ := GetFieldTags(field.Type.Elem()) - nestedType := CreateDynamicStruct(field.Type.Elem(), nestedFieldToJsonTags, depth+1) + tagMaps, _ := GetFieldTags(field.Type.Elem()) + nestedType := CreateDynamicStruct(field.Type.Elem(), tagMaps.FieldToJson, depth+1) fields = append(fields, reflect.StructField{ Name: field.Name, Type: reflect.SliceOf(nestedType), @@ -145,7 +125,7 @@ func MapDynamicToFinal(dynamic any, final any, isNested bool) (uint64, error) { if ok { if len(fieldArr) == 0 { if !isNested { - return 0, ErrNoObjFound + return 0, apiutils.ErrNoObjFound } else { continue } @@ -211,11 +191,11 @@ func ConvertDynamicToTyped[T any](obj any, t reflect.Type) (uint64, T, error) { func GetUniqueConstraint[T any](object T) (uint64, *keyValue, error) { t := reflect.TypeOf(object) - fieldToJsonTags, jsonToDbTags, _, err := GetFieldTags(t) + tagMaps, err := GetFieldTags(t) if err != nil { return 0, nil, err } - jsonTagToValue := GetJsonTagToValues(object, fieldToJsonTags) + jsonTagToValue := GetJsonTagToValues(object, tagMaps.FieldToJson) for jsonName, value := range jsonTagToValue { if jsonName == "gid" { @@ -227,7 +207,7 @@ func GetUniqueConstraint[T any](object T) (uint64, *keyValue, error) { return gid, nil, nil } } - if jsonToDbTags[jsonName] != nil && IsValidUniqueIndex(jsonToDbTags[jsonName].Constraint) { + if tagMaps.JsonToDb[jsonName] != nil && IsValidUniqueIndex(tagMaps.JsonToDb[jsonName].Constraint) { // check if value is zero or nil if value == reflect.Zero(reflect.TypeOf(value)).Interface() || value == nil { continue @@ -236,7 +216,7 @@ func GetUniqueConstraint[T any](object T) (uint64, *keyValue, error) { } } - return 0, nil, fmt.Errorf(NoUniqueConstr, t.Name()) + return 0, nil, fmt.Errorf(apiutils.NoUniqueConstr, t.Name()) } func IsValidUniqueIndex(name string) bool { diff --git a/internal/structreflect/tagparser.go b/internal/structreflect/tagparser.go new file mode 100644 index 0000000..a0d0681 --- /dev/null +++ b/internal/structreflect/tagparser.go @@ -0,0 +1,59 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package structreflect + +import ( + "fmt" + "reflect" + "strings" + + "github.com/hypermodeinc/modusdb/internal/apiutils" +) + +func parseJsonTag(field reflect.StructField) (string, error) { + jsonTag := field.Tag.Get("json") + if jsonTag == "" { + return "", fmt.Errorf("field %s has no json tag", field.Name) + } + return strings.Split(jsonTag, ",")[0], nil +} + +func parseDbTag(field reflect.StructField) *DbTag { + dbConstraintsTag := field.Tag.Get("db") + if dbConstraintsTag == "" { + return nil + } + + dbTag := &DbTag{} + dbTagsSplit := strings.Split(dbConstraintsTag, ",") + for _, tag := range dbTagsSplit { + split := strings.Split(tag, "=") + if split[0] == "constraint" { + dbTag.Constraint = split[1] + } + } + return dbTag +} + +func parseReverseEdgeTag(field reflect.StructField) (string, error) { + reverseEdgeTag := field.Tag.Get("readFrom") + if reverseEdgeTag == "" { + return "", nil + } + + typeAndField := strings.Split(reverseEdgeTag, ",") + if len(typeAndField) != 2 { + return "", fmt.Errorf(`field %s has invalid readFrom tag, expected format is type=,field=`, field.Name) + } + + t := strings.Split(typeAndField[0], "=")[1] + f := strings.Split(typeAndField[1], "=")[1] + return apiutils.GetPredicateName(t, f), nil +} diff --git a/internal/structreflect/tags.go b/internal/structreflect/tags.go new file mode 100644 index 0000000..23f7ac6 --- /dev/null +++ b/internal/structreflect/tags.go @@ -0,0 +1,20 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package structreflect + +type DbTag struct { + Constraint string +} + +type TagMaps struct { + FieldToJson map[string]string + JsonToDb map[string]*DbTag + JsonToReverseEdge map[string]string +} diff --git a/internal/structreflect/value_extractor.go b/internal/structreflect/value_extractor.go new file mode 100644 index 0000000..191ea00 --- /dev/null +++ b/internal/structreflect/value_extractor.go @@ -0,0 +1,28 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package structreflect + +import ( + "reflect" +) + +func GetJsonTagToValues(object any, fieldToJsonTags map[string]string) map[string]any { + values := make(map[string]any) + v := reflect.ValueOf(object) + for v.Kind() == reflect.Ptr { + v = v.Elem() + } + + for fieldName, jsonName := range fieldToJsonTags { + fieldValue := v.FieldByName(fieldName) + values[jsonName] = fieldValue.Interface() + } + return values +} From 205eaf7d2c01fb9bf0b79fa45d27a1008173b6b1 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Mon, 6 Jan 2025 00:20:30 -0800 Subject: [PATCH 22/23] . --- api_mutation_gen.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api_mutation_gen.go b/api_mutation_gen.go index 5c3470b..60d1940 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -46,7 +46,8 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, var nquad *api.NQuad if tagMaps.JsonToReverseEdge[jsonName] != "" { - if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, tagMaps.JsonToReverseEdge); err != nil { + if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, + tagMaps.JsonToReverseEdge); err != nil { return err } continue @@ -71,7 +72,8 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, return err } - uniqueConstraintFound, err = dgraphtypes.HandleConstraints(u, tagMaps.JsonToDb, jsonName, u.ValueType, uniqueConstraintFound) + uniqueConstraintFound, err = dgraphtypes.HandleConstraints(u, tagMaps.JsonToDb, + jsonName, u.ValueType, uniqueConstraintFound) if err != nil { return err } From f24f940dd76a93b5951b2823aa9649f294fd5550 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:59:57 -0800 Subject: [PATCH 23/23] rename internal to api to reflect the functions inside are for the apis specifically --- api.go | 4 ++-- {internal => api}/apiutils/apiutils.go | 0 {internal => api}/dgraphtypes/dgraphtypes.go | 2 +- {internal => api}/mutations/mutations.go | 4 ++-- {internal => api}/querygen/dql_query.go | 0 {internal => api}/structreflect/keyval.go | 0 {internal => api}/structreflect/structreflect.go | 2 +- {internal => api}/structreflect/tagparser.go | 2 +- {internal => api}/structreflect/tags.go | 0 {internal => api}/structreflect/value_extractor.go | 0 api_mutation_gen.go | 8 ++++---- api_mutation_helpers.go | 4 ++-- api_query_execution.go | 6 +++--- api_test.go | 2 +- api_types.go | 4 ++-- 15 files changed, 19 insertions(+), 19 deletions(-) rename {internal => api}/apiutils/apiutils.go (100%) rename {internal => api}/dgraphtypes/dgraphtypes.go (98%) rename {internal => api}/mutations/mutations.go (94%) rename {internal => api}/querygen/dql_query.go (100%) rename {internal => api}/structreflect/keyval.go (100%) rename {internal => api}/structreflect/structreflect.go (99%) rename {internal => api}/structreflect/tagparser.go (96%) rename {internal => api}/structreflect/tags.go (100%) rename {internal => api}/structreflect/value_extractor.go (100%) diff --git a/api.go b/api.go index 0b06df5..b880474 100644 --- a/api.go +++ b/api.go @@ -14,8 +14,8 @@ import ( "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/schema" - "github.com/hypermodeinc/modusdb/internal/apiutils" - "github.com/hypermodeinc/modusdb/internal/structreflect" + "github.com/hypermodeinc/modusdb/api/apiutils" + "github.com/hypermodeinc/modusdb/api/structreflect" ) func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { diff --git a/internal/apiutils/apiutils.go b/api/apiutils/apiutils.go similarity index 100% rename from internal/apiutils/apiutils.go rename to api/apiutils/apiutils.go diff --git a/internal/dgraphtypes/dgraphtypes.go b/api/dgraphtypes/dgraphtypes.go similarity index 98% rename from internal/dgraphtypes/dgraphtypes.go rename to api/dgraphtypes/dgraphtypes.go index 3e4e93a..84a24c6 100644 --- a/internal/dgraphtypes/dgraphtypes.go +++ b/api/dgraphtypes/dgraphtypes.go @@ -17,7 +17,7 @@ import ( "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/types" - "github.com/hypermodeinc/modusdb/internal/structreflect" + "github.com/hypermodeinc/modusdb/api/structreflect" "github.com/twpayne/go-geom" "github.com/twpayne/go-geom/encoding/wkb" ) diff --git a/internal/mutations/mutations.go b/api/mutations/mutations.go similarity index 94% rename from internal/mutations/mutations.go rename to api/mutations/mutations.go index a8bdb7e..3a5e149 100644 --- a/internal/mutations/mutations.go +++ b/api/mutations/mutations.go @@ -17,8 +17,8 @@ import ( "github.com/dgraph-io/dgo/v240/protos/api" "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/schema" - "github.com/hypermodeinc/modusdb/internal/apiutils" - "github.com/hypermodeinc/modusdb/internal/dgraphtypes" + "github.com/hypermodeinc/modusdb/api/apiutils" + "github.com/hypermodeinc/modusdb/api/dgraphtypes" ) func HandleReverseEdge(jsonName string, value reflect.Type, nsId uint64, sch *schema.ParsedSchema, diff --git a/internal/querygen/dql_query.go b/api/querygen/dql_query.go similarity index 100% rename from internal/querygen/dql_query.go rename to api/querygen/dql_query.go diff --git a/internal/structreflect/keyval.go b/api/structreflect/keyval.go similarity index 100% rename from internal/structreflect/keyval.go rename to api/structreflect/keyval.go diff --git a/internal/structreflect/structreflect.go b/api/structreflect/structreflect.go similarity index 99% rename from internal/structreflect/structreflect.go rename to api/structreflect/structreflect.go index ec6b970..fa3b5fd 100644 --- a/internal/structreflect/structreflect.go +++ b/api/structreflect/structreflect.go @@ -14,7 +14,7 @@ import ( "reflect" "strconv" - "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/api/apiutils" ) func GetFieldTags(t reflect.Type) (*TagMaps, error) { diff --git a/internal/structreflect/tagparser.go b/api/structreflect/tagparser.go similarity index 96% rename from internal/structreflect/tagparser.go rename to api/structreflect/tagparser.go index a0d0681..da9224f 100644 --- a/internal/structreflect/tagparser.go +++ b/api/structreflect/tagparser.go @@ -14,7 +14,7 @@ import ( "reflect" "strings" - "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/api/apiutils" ) func parseJsonTag(field reflect.StructField) (string, error) { diff --git a/internal/structreflect/tags.go b/api/structreflect/tags.go similarity index 100% rename from internal/structreflect/tags.go rename to api/structreflect/tags.go diff --git a/internal/structreflect/value_extractor.go b/api/structreflect/value_extractor.go similarity index 100% rename from internal/structreflect/value_extractor.go rename to api/structreflect/value_extractor.go diff --git a/api_mutation_gen.go b/api_mutation_gen.go index 60d1940..6e8426c 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -19,10 +19,10 @@ import ( "github.com/dgraph-io/dgraph/v24/protos/pb" "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/x" - "github.com/hypermodeinc/modusdb/internal/apiutils" - "github.com/hypermodeinc/modusdb/internal/dgraphtypes" - "github.com/hypermodeinc/modusdb/internal/mutations" - "github.com/hypermodeinc/modusdb/internal/structreflect" + "github.com/hypermodeinc/modusdb/api/apiutils" + "github.com/hypermodeinc/modusdb/api/dgraphtypes" + "github.com/hypermodeinc/modusdb/api/mutations" + "github.com/hypermodeinc/modusdb/api/structreflect" ) func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go index aba02bc..f205d8a 100644 --- a/api_mutation_helpers.go +++ b/api_mutation_helpers.go @@ -10,8 +10,8 @@ import ( "github.com/dgraph-io/dgraph/v24/query" "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/worker" - "github.com/hypermodeinc/modusdb/internal/apiutils" - "github.com/hypermodeinc/modusdb/internal/structreflect" + "github.com/hypermodeinc/modusdb/api/apiutils" + "github.com/hypermodeinc/modusdb/api/structreflect" ) func processStructValue(ctx context.Context, value any, n *Namespace) (any, error) { diff --git a/api_query_execution.go b/api_query_execution.go index ec71bc2..4129870 100644 --- a/api_query_execution.go +++ b/api_query_execution.go @@ -15,9 +15,9 @@ import ( "fmt" "reflect" - "github.com/hypermodeinc/modusdb/internal/apiutils" - "github.com/hypermodeinc/modusdb/internal/querygen" - "github.com/hypermodeinc/modusdb/internal/structreflect" + "github.com/hypermodeinc/modusdb/api/apiutils" + "github.com/hypermodeinc/modusdb/api/querygen" + "github.com/hypermodeinc/modusdb/api/structreflect" ) func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, T, error) { diff --git a/api_test.go b/api_test.go index a08b461..3326d86 100644 --- a/api_test.go +++ b/api_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "github.com/hypermodeinc/modusdb" - "github.com/hypermodeinc/modusdb/internal/apiutils" + "github.com/hypermodeinc/modusdb/api/apiutils" ) type User struct { diff --git a/api_types.go b/api_types.go index c81651c..8102771 100644 --- a/api_types.go +++ b/api_types.go @@ -15,8 +15,8 @@ import ( "strings" "github.com/dgraph-io/dgraph/v24/x" - "github.com/hypermodeinc/modusdb/internal/apiutils" - "github.com/hypermodeinc/modusdb/internal/querygen" + "github.com/hypermodeinc/modusdb/api/apiutils" + "github.com/hypermodeinc/modusdb/api/querygen" ) type UniqueField interface {