Fixed preload step on coordinator (#260)

* fixed preload step on coordinator

* Fixed test length check on CI

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.213"
+version = "0.1.214"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
 readme = "Readme.md"

redis_benchmarks_specification/__runner__/runner.py

@@ -255,6 +255,7 @@ def prepare_memtier_benchmark_parameters(
 
     if oss_cluster_api_enabled is True:
         benchmark_command.append("--cluster-mode")
+    logging.info(f"Preparing the benchmark parameters. {benchmark_command}.")
     benchmark_command_str = " ".join(benchmark_command)
     if "arguments" in clientconfig:
         benchmark_command_str = benchmark_command_str + " " + clientconfig["arguments"]

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -1596,11 +1596,13 @@ def data_prepopulation_step(
         (
             _,
             preload_command_str,
+            _,
         ) = prepare_memtier_benchmark_parameters(
             benchmark_config["dbconfig"]["preload_tool"],
             full_benchmark_path,
             port,
             "localhost",
+            None,
             local_benchmark_output_filename,
             False,
         )

utils/tests/test_data/test-suites/generic-touch.yml

@@ -0,0 +1,34 @@
+version: 0.4
+name: memtier_benchmark-1Mkeys-generic-touch-pipeline-10
+description: Runs memtier_benchmark, for a keyspace length of 1M keys focusing on TOUCH performance.
+dbconfig:
+  configuration-parameters:
+    save: '""'
+  check:
+    keyspacelen: 1000000
+  preload_tool:
+    run_image: redislabs/memtier_benchmark:edge
+    tool: memtier_benchmark
+    arguments: '"--data-size" "100" "--command" "SETEX __key__ 3600 __data__" "--command-key-pattern" "P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+  resources:
+    requests:
+      memory: 1g
+tested-commands:
+- touch
+tested-groups:
+- generic
+redis-topologies:
+- oss-standalone
+build-variants:
+- gcc:8.5.0-amd64-debian-buster-default
+- dockerhub
+clientconfig:
+  run_image: redislabs/memtier_benchmark:edge
+  tool: memtier_benchmark
+  arguments: --command "TOUCH __key__" --pipeline 10 --command-key-pattern="R" -c 50 -t 2 --hide-histogram --test-time 180
+  resources:
+    requests:
+      cpus: '2'
+      memory: 2g
+
+priority: 41

utils/tests/test_runner.py

@@ -258,7 +258,7 @@ def test_extract_testsuites():
         ]
     )
     tests = extract_testsuites(args)
-    assert len(tests) == 6
+    assert len(tests) == 7
 
     args = parser.parse_args(
         args=[
@@ -269,7 +269,7 @@ def test_extract_testsuites():
         ]
     )
     tests = extract_testsuites(args)
-    assert len(tests) == 6
+    assert len(tests) == 7
 
     args = parser.parse_args(
         args=[

utils/tests/test_self_contained_coordinator_memtier.py

@@ -226,6 +226,132 @@ def run_coordinator_tests_dockerhub():
     return run_coordinator
 
 
+def test_self_contained_coordinator_dockerhub_preload():
+    try:
+        if run_coordinator_tests_dockerhub():
+            db_port = int(os.getenv("DATASINK_PORT", "6379"))
+            conn = redis.StrictRedis(port=db_port)
+            conn.ping()
+            conn.flushall()
+
+            id = "dockerhub"
+            redis_version = "7.4.0"
+            run_image = f"redis:{redis_version}"
+            build_arch = "amd64"
+            testDetails = {}
+            build_os = "test_build_os"
+            build_stream_fields, result = generate_benchmark_stream_request(
+                id,
+                conn,
+                run_image,
+                build_arch,
+                testDetails,
+                build_os,
+            )
+            build_stream_fields["mnt_point"] = ""
+            if result is True:
+                benchmark_stream_id = conn.xadd(
+                    STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields
+                )
+                logging.info(
+                    "sucessfully requested a new run {}. Stream id: {}".format(
+                        build_stream_fields, benchmark_stream_id
+                    )
+                )
+
+            build_variant_name = "gcc:8.5.0-amd64-debian-buster-default"
+            expected_datapoint_ts = None
+
+            assert conn.exists(STREAM_KEYNAME_NEW_BUILD_EVENTS)
+            assert conn.xlen(STREAM_KEYNAME_NEW_BUILD_EVENTS) > 0
+            running_platform = "fco-ThinkPad-T490"
+
+            build_runners_consumer_group_create(conn, running_platform, "0")
+            datasink_conn = redis.StrictRedis(port=db_port)
+            docker_client = docker.from_env()
+            home = str(Path.home())
+            stream_id = ">"
+            topologies_map = get_topologies(
+                "./redis_benchmarks_specification/setups/topologies/topologies.yml"
+            )
+            # we use a benchmark spec with smaller CPU limit for client given github machines only contain 2 cores
+            # and we need 1 core for DB and another for CLIENT
+            testsuite_spec_files = [
+                "./utils/tests/test_data/test-suites/generic-touch.yml"
+            ]
+            defaults_filename = "./utils/tests/test_data/test-suites/defaults.yml"
+            (
+                _,
+                _,
+                default_metrics,
+                _,
+                _,
+                _,
+            ) = get_defaults(defaults_filename)
+
+            (
+                result,
+                stream_id,
+                number_processed_streams,
+                num_process_test_suites,
+            ) = self_contained_coordinator_blocking_read(
+                conn,
+                True,
+                docker_client,
+                home,
+                stream_id,
+                datasink_conn,
+                testsuite_spec_files,
+                topologies_map,
+                running_platform,
+                False,
+                [],
+                "",
+                0,
+                6399,
+                1,
+                False,
+                5,
+                default_metrics,
+                "amd64",
+                None,
+                0,
+                10000,
+                "unstable",
+                "",
+                True,
+                False,
+            )
+
+            assert result == True
+            assert number_processed_streams == 1
+            assert num_process_test_suites == 1
+            by_version_key = f"ci.benchmarks.redislabs/ci/redis/redis/memtier_benchmark-1Mkeys-generic-touch-pipeline-10/by.version/{redis_version}/benchmark_end/oss-standalone/memory_maxmemory"
+            assert datasink_conn.exists(by_version_key)
+            rts = datasink_conn.ts()
+            # check we have by version metrics
+            assert "version" in rts.info(by_version_key).labels
+            assert redis_version == rts.info(by_version_key).labels["version"]
+
+            # get all keys
+            all_keys = datasink_conn.keys("*")
+            by_hash_keys = []
+            for key in all_keys:
+                if "/by.hash/" in key.decode():
+                    by_hash_keys.append(key)
+
+            # ensure we have by hash keys
+            assert len(by_hash_keys) > 0
+            for hash_key in by_hash_keys:
+                # ensure we have both version and hash info on the key
+                assert "version" in rts.info(hash_key).labels
+                assert "hash" in rts.info(hash_key).labels
+                assert redis_version == rts.info(hash_key).labels["version"]
+
+    except redis.exceptions.ConnectionError:
+        pass
+
+
 def test_self_contained_coordinator_dockerhub():
     try:
         if run_coordinator_tests_dockerhub():