diff --git a/README.md b/README.md index 47dbf35..612d6fb 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ JSON.DEL JSON.FORGET JSON.GET JSON.MGET -JSON.MSET +JSON.MSET (#16) JSON.NUMINCRBY JSON.NUMMULTBY JSON.OBJLEN @@ -116,4 +116,3 @@ JSON.STRLEN JSON.TOGGLE JSON.TYPE ``` - diff --git a/src/json/json.cc b/src/json/json.cc index 262abe4..7158d2c 100644 --- a/src/json/json.cc +++ b/src/json/json.cc @@ -2393,6 +2393,26 @@ int scdtype_aux_load(ValkeyModuleIO *ctx, int encver, int when) { return VALKEYMODULE_OK; } +/* + * Stub for ftindex0 data type. There is one integer of 0's. + * There's an 18, a 19 and a 20. They don't appear to be any different when the data is empty :) + */ +#define FTINDEX_ENCVER 20 +int ftindex_aux_load(ValkeyModuleIO *ctx, int encver, int when) { + VALKEYMODULE_NOT_USED(encver); + VALKEYMODULE_NOT_USED(when); + if (!loadUnsigned(ctx, "ftindex")) return VALKEYMODULE_ERR; + return VALKEYMODULE_OK; +} + +#define GRAPHDT_ENCVER 11 +int graphdt_aux_load(ValkeyModuleIO *ctx, int encver, int when) { + VALKEYMODULE_NOT_USED(encver); + VALKEYMODULE_NOT_USED(when); + if (!loadUnsigned(ctx, "graphdt")) return VALKEYMODULE_ERR; + return VALKEYMODULE_OK; +} + #define GEARSDT_ENCVER 3 int gearsdt_aux_load(ValkeyModuleIO *ctx, int encver, int when) { VALKEYMODULE_NOT_USED(encver); @@ -2527,6 +2547,8 @@ extern "C" int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx) { * Now create the stub datatypes for search */ if (!install_stub(ctx, "scdtype00", SCDTYPE_ENCVER, scdtype_aux_load)) return VALKEYMODULE_ERR; + if (!install_stub(ctx, "ft_index0", FTINDEX_ENCVER, ftindex_aux_load)) return VALKEYMODULE_ERR; + if (!install_stub(ctx, "graphdata", GRAPHDT_ENCVER, graphdt_aux_load)) return VALKEYMODULE_ERR; if (!install_stub(ctx, "GEARS_DT0", GEARSDT_ENCVER, gearsdt_aux_load)) return VALKEYMODULE_ERR; if (!install_stub(ctx, "GEAR_REQ0", GEARSRQ_ENCVER, gearsrq_aux_load)) return VALKEYMODULE_ERR; diff --git a/tst/integration/rdb_rejson/README.md b/tst/integration/rdb_rejson/README.md new file mode 100644 index 0000000..7bd74c0 --- /dev/null +++ b/tst/integration/rdb_rejson/README.md @@ -0,0 +1,24 @@ +#### How to create rdb file for a new ReJSON release? + +e.g., testing RDB compatibility with rejson 2.2.0. + +1. Run docker image redis-stack: + ```text + docker run -d -p 6379:6379 --name rejson redislabs/rejson:2.2.0 + ``` +2. Load store.json and create a key named "store": + ```text + python3 utils/load_1file_hostport.py tst/integration/data/store.json store + ``` +3. Save rdb: + ```text + valkey-cli save + ``` +4. Copy out the RDB file: + ```text + docker cp $(docker ps -q):/data/dump.rdb tst/integration/rdb_rejson/rejson-.rdb + ``` +5. run test_json_rdb_import.py: + ```text + TEST_PATTERN=test_import_rejson_rdbs make test + ``` diff --git a/tst/integration/rdb_rejson/rejson-1.0.8.rdb b/tst/integration/rdb_rejson/rejson-1.0.8.rdb new file mode 100644 index 0000000..d9bda11 Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-1.0.8.rdb differ diff --git a/tst/integration/rdb_rejson/rejson-2.0.11.rdb b/tst/integration/rdb_rejson/rejson-2.0.11.rdb new file mode 100644 index 0000000..ddc4817 Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-2.0.11.rdb differ diff --git a/tst/integration/rdb_rejson/rejson-2.0.6.rdb b/tst/integration/rdb_rejson/rejson-2.0.6.rdb new file mode 100644 index 0000000..03fcd3f Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-2.0.6.rdb differ diff --git a/tst/integration/rdb_rejson/rejson-2.0.7.rdb b/tst/integration/rdb_rejson/rejson-2.0.7.rdb new file mode 100644 index 0000000..4e30e37 Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-2.0.7.rdb differ diff --git a/tst/integration/rdb_rejson/rejson-2.0.8.rdb b/tst/integration/rdb_rejson/rejson-2.0.8.rdb new file mode 100644 index 0000000..df7787b Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-2.0.8.rdb differ diff --git a/tst/integration/rdb_rejson/rejson-2.2.0.rdb b/tst/integration/rdb_rejson/rejson-2.2.0.rdb new file mode 100644 index 0000000..50a2d1e Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-2.2.0.rdb differ diff --git a/tst/integration/rdb_rejson/rejson-2.6.12.rdb b/tst/integration/rdb_rejson/rejson-2.6.12.rdb new file mode 100644 index 0000000..eb9e8e2 Binary files /dev/null and b/tst/integration/rdb_rejson/rejson-2.6.12.rdb differ diff --git a/tst/integration/run.sh b/tst/integration/run.sh index f454f9f..7cb2bb5 100755 --- a/tst/integration/run.sh +++ b/tst/integration/run.sh @@ -18,8 +18,9 @@ fi DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" cd "${DIR}" -export MODULE_PATH=$2/build/src/libjson.so -echo "Running integration tests against Valkey version ${SERVER_VERSION}" +export SOURCE_DIR=$2 +export MODULE_PATH=${SOURCE_DIR}/build/src/libjson.so +echo "Running integration tests against Valkey version $SERVER_VERSION" if [[ ! -z "${TEST_PATTERN}" ]] ; then export TEST_PATTERN="-k ${TEST_PATTERN}" diff --git a/tst/integration/test_json_basic.py b/tst/integration/test_json_basic.py index 54489a2..b487ab7 100644 --- a/tst/integration/test_json_basic.py +++ b/tst/integration/test_json_basic.py @@ -13,6 +13,8 @@ from math import isclose, isnan, isinf, frexp from json_test_case import JsonTestCase +logging.basicConfig(level=logging.DEBUG) + DATA_ORGANISM = ''' { "heavy_animal" : 200, @@ -1693,7 +1695,7 @@ def test_json_nummultby_large_numbers(self): v.decode() == val or v.decode() == val_alt) v = client.execute_command( 'JSON.NUMMULTBY', wikipedia, '.foo', mult) - # print("DEBUG val: %s, mult: %f, v: %s, exp: %s" %(val, mult, v.decode(), exp)) + # logging.debug("DEBUG val: %s, mult: %f, v: %s, exp: %s" %(val, mult, v.decode(), exp)) assert v is not None and v.decode() == exp or v.decode() == exp_alt v = client.execute_command( 'JSON.GET', wikipedia, '.foo') diff --git a/tst/integration/test_rdb.py b/tst/integration/test_rdb.py index da8b3d8..bdad286 100644 --- a/tst/integration/test_rdb.py +++ b/tst/integration/test_rdb.py @@ -1,17 +1,9 @@ -from utils_json import DEFAULT_MAX_PATH_LIMIT, \ - DEFAULT_STORE_PATH -from valkey.exceptions import ResponseError, NoPermissionError -from valkeytests.conftest import resource_port_tracker -import pytest -import glob -import logging -import os -import random -import struct -import json -from math import isclose, isnan, isinf, frexp +from utils_json import DEFAULT_MAX_PATH_LIMIT, DEFAULT_STORE_PATH from json_test_case import JsonTestCase +from valkeytests.conftest import resource_port_tracker +import logging, os, pathlib +logging.basicConfig(level=logging.DEBUG) class TestRdb(JsonTestCase): @@ -42,3 +34,31 @@ def test_rdb_saverestore(self): assert True == client.execute_command('save') client.execute_command('FLUSHDB') assert b'OK' == client.execute_command('DEBUG', 'RELOAD', 'NOSAVE') + + def test_import_rejson_rdbs(self): + ''' + Verify we can load RDBs generated from ReJSON. + Each RDB file contains JSON key "store" (data/store.json). + ''' + self.load_rdbs_from_dir('rdb_rejson') + + def load_rdbs_from_dir(self, dir): + src_dir = os.getenv('SOURCE_DIR') + rdb_dir = f"{src_dir}/tst/integration/{dir}" + assert os.path.exists(rdb_dir) + for (dirpath, dirnames, filenames) in os.walk(rdb_dir): + for filename in filenames: + if pathlib.Path(filename).suffix == '.rdb': + file_path = os.path.join(dirpath, filename) + self.load_rdb_file(file_path, filename) + + def load_rdb_file(self, rdb_path, rdb_name): + new_path = os.path.join(self.testdir, rdb_name) + os.system(f"cp -f {rdb_path} {new_path}") + logging.info(f"Loading RDB file {new_path}") + self.client.execute_command(f"config set dbfilename {rdb_name}") + self.client.execute_command("debug reload nosave") + self.verify_keys_in_rejson_rdb() + + def verify_keys_in_rejson_rdb(self): + assert b'["The World Almanac and Book of Facts 2021"]' == self.client.execute_command('json.get', 'store', '$..books[?(@.price>10&&@.price<22&&@.isbn)].title') diff --git a/utils/load_1file_hostport.py b/utils/load_1file_hostport.py new file mode 100755 index 0000000..0a00a8d --- /dev/null +++ b/utils/load_1file_hostport.py @@ -0,0 +1,39 @@ +#!python3 +# +# Load a JSON file and create a key. +# Usage: +# [HOST=] [PORT=] [SSL=] python3 load_1file_hostport.py +# +# e.g. +# python3 load_1file_hostport.py ../amztests/data/wikipedia.json wikipedia +# PORT=6380 python3 load_1file_hostport.py ../amztests/data/wikipedia.json wikipedia +# +import redis, sys, os, logging +from redis.exceptions import ResponseError, ConnectionError, TimeoutError + +if len(sys.argv) < 3: + print("Usage: [HOST=] [PORT=] [SSL=] python3 load_1file_hostport.py ") + exit(1) + +host = os.getenv('HOST', 'localhost') +port = os.getenv('PORT', '6379') +ssl = os.getenv('SSL', 'False') +is_ssl = (ssl == 'True') +json_file_path = sys.argv[1] +key = sys.argv[2] + +r = redis.Redis(host=host, port=port, ssl=is_ssl, socket_timeout=3) +try: + r.ping() + logging.info(f"Connected to valkey {host}:{port}, ssl: {is_ssl}") +except (ConnectionError, TimeoutError): + logging.error(f"Failed to connect to valkey {host}:{port}, ssl: {is_ssl}") + exit(1) + +def load_file(json_file_path, key): + with open(json_file_path, 'r') as f: + data = f.read() + r.execute_command('json.set', key, '.', data) + logging.info("Created key %s" %key) + +load_file(json_file_path, key)