Fix #19 - RDB compatibility with ReJSON and add compatibility test

Signed-off-by: Joe Hu <[email protected]>

README.md

@@ -82,7 +82,7 @@ JSON.DEL
 JSON.FORGET
 JSON.GET
 JSON.MGET
-JSON.MSET
+JSON.MSET (#16)
 JSON.NUMINCRBY
 JSON.NUMMULTBY
 JSON.OBJLEN
@@ -94,4 +94,3 @@ JSON.STRLEN
 JSON.TOGGLE
 JSON.TYPE
 ```
-

src/json/json.cc

@@ -2594,6 +2594,26 @@ int scdtype_aux_load(ValkeyModuleIO *ctx, int encver, int when) {
     return VALKEYMODULE_OK;
 }
 
+/*
+ * Stub for ftindex0 data type. There is one integer of 0's.
+ * There's an 18, a 19 and a 20. They don't appear to be any different when the data is empty :)
+ */
+#define FTINDEX_ENCVER 20
+int ftindex_aux_load(ValkeyModuleIO *ctx, int encver, int when) {
+    VALKEYMODULE_NOT_USED(encver);
+    VALKEYMODULE_NOT_USED(when);
+    if (!loadUnsigned(ctx, "ftindex")) return VALKEYMODULE_ERR;
+    return VALKEYMODULE_OK;
+}
+
+#define GRAPHDT_ENCVER 11
+int graphdt_aux_load(ValkeyModuleIO *ctx, int encver, int when) {
+    VALKEYMODULE_NOT_USED(encver);
+    VALKEYMODULE_NOT_USED(when);
+    if (!loadUnsigned(ctx, "graphdt")) return VALKEYMODULE_ERR;
+    return VALKEYMODULE_OK;
+}
+
 #define GEARSDT_ENCVER 3
 int gearsdt_aux_load(ValkeyModuleIO *ctx, int encver, int when) {
     VALKEYMODULE_NOT_USED(encver);
@@ -2728,6 +2748,8 @@ extern "C" int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx) {
      * Now create the stub datatypes for search
      */
     if (!install_stub(ctx, "scdtype00", SCDTYPE_ENCVER, scdtype_aux_load)) return VALKEYMODULE_ERR;
+    if (!install_stub(ctx, "ft_index0", FTINDEX_ENCVER, ftindex_aux_load)) return VALKEYMODULE_ERR;
+    if (!install_stub(ctx, "graphdata", GRAPHDT_ENCVER, graphdt_aux_load)) return VALKEYMODULE_ERR;
     if (!install_stub(ctx, "GEARS_DT0", GEARSDT_ENCVER, gearsdt_aux_load)) return VALKEYMODULE_ERR;
     if (!install_stub(ctx, "GEAR_REQ0", GEARSRQ_ENCVER, gearsrq_aux_load)) return VALKEYMODULE_ERR;
 

tst/integration/rdb_rejson/README.txt

@@ -0,0 +1,14 @@
+How to create rdb file for a new ReJSON release?
+
+e.g., testing RDB compatibility with rejson 2.2.0.
+
+1. Run docker image redis-stack:
+    docker run -d -p 6379:6379 --name rejson redislabs/rejson:2.2.0
+2. Load store.json and create a key named "store":
+    python3 utils/load_1file_hostport.py tst/integration/data/store.json store
+3. Save rdb:
+    valkey-cli save
+4. Copy out the RDB file:
+    docker cp $(docker ps -q):/data/dump.rdb tst/integration/rdb_rejson/rejson-<version>.rdb
+5. run test_json_rdb_import.py:
+    TEST_PATTERN=test_import_rejson_rdbs make test

tst/integration/run.sh

@@ -18,7 +18,8 @@ fi
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd "${DIR}"
 
-export MODULE_PATH=$2/build/src/libjson.so 
+export SOURCE_DIR=$2
+export MODULE_PATH=${SOURCE_DIR}/build/src/libjson.so
 echo "Running integration tests against Valkey version $SERVER_VERSION"
 
 if [[ ! -z "${TEST_PATTERN}" ]] ; then

tst/integration/test_json_basic.py

@@ -13,6 +13,8 @@
 from math import isclose, isnan, isinf, frexp
 from json_test_case import JsonTestCase
 
+logging.basicConfig(level=logging.DEBUG)
+
 DATA_ORGANISM = '''
             {
                 "heavy_animal" : 200,
@@ -1693,7 +1695,7 @@ def test_json_nummultby_large_numbers(self):
                 v.decode() == val or v.decode() == val_alt)
             v = client.execute_command(
                 'JSON.NUMMULTBY', wikipedia, '.foo', mult)
-            # print("DEBUG val: %s, mult: %f, v: %s, exp: %s" %(val, mult, v.decode(), exp))
+            # logging.debug("DEBUG val: %s, mult: %f, v: %s, exp: %s" %(val, mult, v.decode(), exp))
             assert v is not None and v.decode() == exp or v.decode() == exp_alt
             v = client.execute_command(
                 'JSON.GET', wikipedia, '.foo')

tst/integration/test_rdb.py

@@ -1,17 +1,9 @@
-from utils_json import DEFAULT_MAX_PATH_LIMIT, \
-    DEFAULT_STORE_PATH
-from valkey.exceptions import ResponseError, NoPermissionError
-from valkeytests.conftest import resource_port_tracker
-import pytest
-import glob
-import logging
-import os
-import random
-import struct
-import json
-from math import isclose, isnan, isinf, frexp
+from utils_json import DEFAULT_MAX_PATH_LIMIT, DEFAULT_STORE_PATH
 from json_test_case import JsonTestCase
+from valkeytests.conftest import resource_port_tracker
+import logging, os, pathlib
 
+logging.basicConfig(level=logging.DEBUG)
 
 class TestRdb(JsonTestCase):
 
@@ -42,3 +34,31 @@ def test_rdb_saverestore(self):
         assert True == client.execute_command('save')
         client.execute_command('FLUSHDB')
         assert b'OK' == client.execute_command('DEBUG', 'RELOAD', 'NOSAVE')
+
+    def test_import_rejson_rdbs(self):
+        '''
+        Verify we can load RDBs generated from ReJSON.
+        Each RDB file contains JSON key "store" (data/store.json).
+        '''
+        self.load_rdbs_from_dir('rdb_rejson')
+
+    def load_rdbs_from_dir(self, dir):
+        src_dir = os.getenv('SOURCE_DIR')
+        rdb_dir = f"{src_dir}/tst/integration/{dir}"
+        assert os.path.exists(rdb_dir)
+        for (dirpath, dirnames, filenames) in os.walk(rdb_dir):
+            for filename in filenames:
+                if pathlib.Path(filename).suffix == '.rdb':
+                    file_path = os.path.join(dirpath, filename)
+                    self.load_rdb_file(file_path, filename)
+
+    def load_rdb_file(self, rdb_path, rdb_name):
+        new_path = os.path.join(self.testdir, rdb_name)
+        os.system(f"cp -f {rdb_path} {new_path}")
+        logging.info(f"Loading RDB file {new_path}")
+        self.client.execute_command(f"config set dbfilename {rdb_name}")
+        self.client.execute_command("debug reload nosave")
+        self.verify_keys_in_rejson_rdb()
+
+    def verify_keys_in_rejson_rdb(self):
+        assert b'["The World Almanac and Book of Facts 2021"]' == self.client.execute_command('json.get', 'store', '$..books[?(@.price>10&&@.price<22&&@.isbn)].title')

utils/load_1file_hostport.py

@@ -0,0 +1,39 @@
+#!python3
+#
+# Load a JSON file and create a key.
+# Usage:
+# [HOST=<host>] [PORT=<port>] [SSL=<ssl>] python3 load_1file_hostport.py <path_to_json> <key>
+#
+# e.g.
+# python3 load_1file_hostport.py ../amztests/data/wikipedia.json wikipedia
+# PORT=6380 python3 load_1file_hostport.py ../amztests/data/wikipedia.json wikipedia
+#
+import redis, sys, os, logging
+from redis.exceptions import ResponseError, ConnectionError, TimeoutError
+
+if len(sys.argv) < 3:
+    print("Usage: [HOST=<host>] [PORT=<port>] [SSL=<ssl>] python3 load_1file_hostport.py <path_to_json> <redis_key>")
+    exit(1)
+
+host = os.getenv('HOST', 'localhost')
+port = os.getenv('PORT', '6379')
+ssl = os.getenv('SSL', 'False')
+is_ssl = (ssl == 'True')
+json_file_path = sys.argv[1]
+key = sys.argv[2]
+
+r = redis.Redis(host=host, port=port, ssl=is_ssl, socket_timeout=3)
+try:
+    r.ping()
+    logging.info(f"Connected to valkey {host}:{port}, ssl: {is_ssl}")
+except (ConnectionError, TimeoutError):
+    logging.error(f"Failed to connect to valkey {host}:{port}, ssl: {is_ssl}")
+    exit(1)
+
+def load_file(json_file_path, key):
+    with open(json_file_path, 'r') as f:
+        data = f.read()
+    r.execute_command('json.set', key, '.', data)
+    logging.info("Created key %s" %key)
+
+load_file(json_file_path, key)