Merge pull request #1357 from DiceDB/rem-encoding

Bloom Filter and Type Standardization

integration_tests/commands/http/bloom_test.go

@@ -3,6 +3,7 @@ package http
 import (
 	"testing"
 
+	diceerrors "github.com/dicedb/dice/internal/errors"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -82,7 +83,7 @@ func TestBloomFilter(t *testing.T) {
 					Body:    map[string]interface{}{"key": "bf", "values": []interface{}{0.01, 2000}},
 				},
 			},
-			expected: []interface{}{"OK", "ERR item exists"},
+			expected: []interface{}{"OK", diceerrors.ErrKeyExists.Error()},
 		},
 	}
 
@@ -199,7 +200,7 @@ func TestBFEdgeCasesAndErrors(t *testing.T) {
 					Body:    map[string]interface{}{"key": "bf"},
 				},
 			},
-			expected: []interface{}{"ERR not found"},
+			expected: []interface{}{diceerrors.ErrKeyNotFound.Error()},
 		},
 		{
 			name: "BF.RESERVE with a very high error rate",
@@ -281,7 +282,7 @@ func TestBFEdgeCasesAndErrors(t *testing.T) {
 					Body:    map[string]interface{}{"key": "bf", "values": []interface{}{0.01, 2000}},
 				},
 			},
-			expected: []interface{}{"OK", "ERR item exists"},
+			expected: []interface{}{"OK", diceerrors.ErrKeyExists.Error()},
 		},
 		{
 			name: "BF.INFO after multiple additions",

integration_tests/commands/http/jsondebug_test.go

@@ -16,7 +16,7 @@ func TestJSONDEBUG(t *testing.T) {
 				{Command: "JSON.SET", Body: map[string]interface{}{"key": "k1", "path": "$", "json": map[string]interface{}{"a": 1}}},
 				{Command: "JSON.DEBUG", Body: map[string]interface{}{"values": []interface{}{"MEMORY", "k1"}}},
 			},
-			expected: []interface{}{"OK", float64(89)},
+			expected: []interface{}{"OK", float64(72)},
 		},
 		{
 			name: "jsondebug with a valid path",

integration_tests/commands/resp/bloom_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	diceerrors "github.com/dicedb/dice/internal/errors"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -42,7 +43,7 @@ func TestBFReserveAddInfoExists(t *testing.T) {
 		{
 			name:    "BF.RESERVE on existent filter returns error",
 			cmds:    []string{"BF.RESERVE bf 0.01 1000", "BF.RESERVE bf 0.01 1000"},
-			expect:  []interface{}{"OK", "ERR item exists"},
+			expect:  []interface{}{"OK", diceerrors.ErrKeyExists.Error()},
 			delays:  []time.Duration{0, 0},
 			cleanUp: []string{"DEL bf"},
 		},
@@ -135,7 +136,7 @@ func TestBFEdgeCasesAndErrors(t *testing.T) {
 		{
 			name:    "BF.INFO on a non-existent filter",
 			cmds:    []string{"BF.INFO bf"},
-			expect:  []interface{}{"ERR not found"},
+			expect:  []interface{}{diceerrors.ErrKeyNotFound.Error()},
 			delays:  []time.Duration{0},
 			cleanUp: []string{"del bf"},
 		},
@@ -170,7 +171,7 @@ func TestBFEdgeCasesAndErrors(t *testing.T) {
 		{
 			name:    "BF.RESERVE with duplicate filter name",
 			cmds:    []string{"BF.RESERVE bf 0.01 1000", "BF.RESERVE bf 0.01 2000"},
-			expect:  []interface{}{"OK", "ERR item exists"},
+			expect:  []interface{}{"OK", diceerrors.ErrKeyExists.Error()},
 			delays:  []time.Duration{0, 0},
 			cleanUp: []string{"del bf"},
 		},

integration_tests/commands/resp/jsondebug_test.go

@@ -23,7 +23,7 @@ func TestJSONDEBUG(t *testing.T) {
 				`JSON.SET k1 $ {"a":1}`,
 				"JSON.DEBUG MEMORY k1",
 			},
-			expected: []interface{}{"OK", int64(89)},
+			expected: []interface{}{"OK", int64(72)},
 		},
 		{
 			name: "jsondebug with a valid path",

integration_tests/commands/websocket/bloom_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	diceerrors "github.com/dicedb/dice/internal/errors"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -37,7 +38,7 @@ func TestBFReserveAddInfoExists(t *testing.T) {
 		{
 			name:    "BF.RESERVE on existent filter returns error",
 			cmds:    []string{"BF.RESERVE bf 0.01 1000", "BF.RESERVE bf 0.01 1000"},
-			expect:  []interface{}{"OK", "ERR item exists"},
+			expect:  []interface{}{"OK", diceerrors.ErrKeyExists.Error()},
 			cleanUp: []string{"DEL bf"},
 		},
 	}
@@ -126,7 +127,7 @@ func TestBFEdgeCasesAndErrors(t *testing.T) {
 		{
 			name:    "BF.INFO on a non-existent filter",
 			cmds:    []string{"BF.INFO bf"},
-			expect:  []interface{}{"ERR not found"},
+			expect:  []interface{}{diceerrors.ErrKeyNotFound.Error()},
 			delays:  []time.Duration{0},
 			cleanUp: []string{"del bf"},
 		},
@@ -161,7 +162,7 @@ func TestBFEdgeCasesAndErrors(t *testing.T) {
 		{
 			name:    "BF.RESERVE with duplicate filter name",
 			cmds:    []string{"BF.RESERVE bf 0.01 1000", "BF.RESERVE bf 0.01 2000"},
-			expect:  []interface{}{"OK", "ERR item exists"},
+			expect:  []interface{}{"OK", diceerrors.ErrKeyExists.Error()},
 			delays:  []time.Duration{0, 0},
 			cleanUp: []string{"del bf"},
 		},

integration_tests/commands/websocket/jsondebug_test.go

@@ -30,7 +30,7 @@ func TestJSONDEBUG(t *testing.T) {
 				`JSON.SET k1 $ {"a":1}`,
 				"JSON.DEBUG MEMORY k1",
 			},
-			expected: []interface{}{"OK", float64(89)},
+			expected: []interface{}{"OK", float64(72)},
 		},
 		{
 			name: "jsondebug with a valid path",

internal/errors/migrated_errors.go

@@ -33,6 +33,7 @@ var (
 	ErrInvalidIPAddress           = errors.New("invalid IP address")
 	ErrInvalidFingerprint         = errors.New("invalid fingerprint")
 	ErrKeyDoesNotExist            = errors.New("ERR could not perform this operation on a key that doesn't exist")
+	ErrKeyExists                  = errors.New("ERR key exists")
 
 	// Error generation functions for specific error messages with dynamic parameters.
 	ErrWrongArgumentCount = func(command string) error {

internal/eval/bloom_test.go

@@ -100,19 +100,19 @@ func TestGetOrCreateBloomFilter(t *testing.T) {
 	opts := defaultBloomOpts()
 
 	// Should create a new filter under the key `key`.
-	bloom, err := getOrCreateBloomFilter(key, store, opts)
+	bloom, err := GetOrCreateBloomFilter(key, store, opts)
 	if bloom == nil || err != nil {
 		t.Errorf("nil bloom or non-nil error returned while creating new filter - key: %s, opts: %+v, err: %v", key, opts, err)
 	}
 
 	// Should get the filter (which was created above)
-	bloom, err = getOrCreateBloomFilter(key, store, opts)
+	bloom, err = GetOrCreateBloomFilter(key, store, opts)
 	if bloom == nil || err != nil {
 		t.Errorf("nil bloom or non-nil error returned while fetching existing filter - key: %s, opts: %+v, err: %v", key, opts, err)
 	}
 
 	// Should get the filter with nil opts
-	bloom, err = getOrCreateBloomFilter(key, store, nil)
+	bloom, err = GetOrCreateBloomFilter(key, store, nil)
 	if bloom == nil || err != nil {
 		t.Errorf("nil bloom or non-nil error returned while fetching existing filter - key: %s, opts: %+v, err: %v", key, opts, err)
 	}
@@ -122,7 +122,7 @@ func TestUpdateIndexes(t *testing.T) {
 	// Create a value, default opts and initialize all params of the filter
 	value := "hello"
 	opts := defaultBloomOpts()
-	bloom := newBloomFilter(opts)
+	bloom := NewBloomFilter(opts)
 
 	err := opts.updateIndexes(value)
 	if err != nil {

internal/eval/bytearray.go

@@ -36,7 +36,7 @@ func NewByteArrayFromObj(obj *object.Obj) (*ByteArray, error) {
 }
 
 func getValueAsByteSlice(obj *object.Obj) ([]byte, error) {
-	oType := object.ExtractType(obj)
+	oType := obj.Type
 	switch oType {
 	case object.ObjTypeInt:
 		return []byte(strconv.FormatInt(obj.Value.(int64), 10)), nil
@@ -81,7 +81,7 @@ func getByteArrayValueAsByteSlice(obj *object.Obj) ([]byte, error) {
 }
 
 // ByteSliceToObj converts a byte slice to an Obj of the specified type and encoding
-func ByteSliceToObj(store *dstore.Store, oldObj *object.Obj, b []byte, objType uint8) (*object.Obj, error) {
+func ByteSliceToObj(store *dstore.Store, oldObj *object.Obj, b []byte, objType object.ObjectType) (*object.Obj, error) {
 	switch objType {
 	case object.ObjTypeInt:
 		return ByteSliceToIntObj(store, oldObj, b)

internal/eval/dump_restore.go

@@ -24,10 +24,12 @@ func rdbDeserialize(data []byte) (*object.Obj, error) {
 	if err != nil {
 		return nil, err
 	}
-	objType, err := buf.ReadByte()
+	_oType, err := buf.ReadByte()
 	if err != nil {
 		return nil, err
 	}
+
+	objType := object.ObjectType(_oType)
 	switch objType {
 	case object.ObjTypeString:
 		value, err = readString(buf)
@@ -112,7 +114,7 @@ func readSet(buf *bytes.Reader) (interface{}, error) {
 func rdbSerialize(obj *object.Obj) ([]byte, error) {
 	var buf bytes.Buffer
 	buf.WriteByte(0x09)
-	buf.WriteByte(obj.Type)
+	buf.WriteByte(byte(obj.Type))
 	switch obj.Type {
 	case object.ObjTypeString:
 		str, ok := obj.Value.(string)

internal/eval/eval.go

@@ -165,16 +165,7 @@ func evalMSET(args []string, store *dstore.Store) []byte {
 	insertMap := make(map[string]*object.Obj, len(args)/2)
 	for i := 0; i < len(args); i += 2 {
 		key, value := args[i], args[i+1]
-		oType := deduceType(value)
-		var storedValue interface{}
-		switch oType {
-		case object.ObjTypeInt:
-			storedValue, _ = strconv.ParseInt(value, 10, 64)
-		case object.ObjTypeString:
-			storedValue = value
-		default:
-			return clientio.Encode(fmt.Errorf("ERR unsupported type: %d", oType), false)
-		}
+		storedValue, oType := getRawStringOrInt(value)
 		insertMap[key] = store.NewObj(storedValue, exDurationMs, oType)
 	}
 

internal/eval/eval_test.go

@@ -2995,7 +2995,7 @@ func testEvalPFADD(t *testing.T, store *dstore.Store) {
 			name: "PFADD Incorrect type provided",
 			setup: func() {
 				key, value := "EXISTING_KEY", "VALUE"
-				oType := deduceType(value)
+				_, oType := getRawStringOrInt(value)
 				var exDurationMs int64 = -1
 				keepttl := false
 
@@ -4830,7 +4830,7 @@ func testEvalDebug(t *testing.T, store *dstore.Store) {
 				store.Put(key, obj)
 			},
 			input:          []string{"MEMORY", "EXISTING_KEY"},
-			migratedOutput: EvalResponse{Result: 89, Error: nil},
+			migratedOutput: EvalResponse{Result: 72, Error: nil},
 		},
 
 		"root path": {
@@ -4843,7 +4843,7 @@ func testEvalDebug(t *testing.T, store *dstore.Store) {
 				store.Put(key, obj)
 			},
 			input:          []string{"MEMORY", "EXISTING_KEY", "$"},
-			migratedOutput: EvalResponse{Result: 89, Error: nil},
+			migratedOutput: EvalResponse{Result: 72, Error: nil},
 		},
 
 		"invalid path": {
@@ -6719,7 +6719,7 @@ func testEvalAPPEND(t *testing.T, store *dstore.Store) {
 			migratedOutput: EvalResponse{Result: 3, Error: nil},
 			validator: func(output []byte) {
 				obj := store.Get("key")
-				oType := object.ExtractType(obj)
+				oType := obj.Type
 				if oType != object.ObjTypeInt {
 					t.Errorf("unexpected encoding")
 				}
@@ -6774,7 +6774,7 @@ func testEvalAPPEND(t *testing.T, store *dstore.Store) {
 			migratedOutput: EvalResponse{Result: 2, Error: nil},
 			validator: func(output []byte) {
 				obj := store.Get("key")
-				oType := object.ExtractType(obj)
+				oType := obj.Type
 				if oType != object.ObjTypeString {
 					t.Errorf("unexpected encoding")
 				}
@@ -9022,7 +9022,7 @@ func testEvalBFINFO(t *testing.T, store *dstore.Store) {
 		{
 			name:           "BF.INFO on non-existent filter",
 			input:          []string{"nonExistentFilter"},
-			migratedOutput: EvalResponse{Result: nil, Error: errors.New("ERR not found")},
+			migratedOutput: EvalResponse{Result: nil, Error: diceerrors.ErrKeyNotFound},
 		},
 	}