Increase the RFS default scale to 5 (opensearch-project#1057)

* Increase the RFS default scale to 5 After starting backfill it appears to be slow because a single worker node needs to spin up and down to process the default index with 5 shards. By increasing the default this should move at least an index at a time causing the overall process to move much faster. Signed-off-by: Peter Nied <[email protected]>
lewijacn · Oct 9, 2024 · bee316d · bee316d
1 parent 7307579
commit bee316d
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 13 deletions.
diff --git a/...ture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md b/...ture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md
@@ -1,23 +1,24 @@
 # Console_link Library
 
-- [Services.yaml spec](#servicesyaml-spec)
+- [Console\_link Library](#console_link-library)
+  - [Services.yaml spec](#servicesyaml-spec)
     - [Cluster](#cluster)
     - [Metrics Source](#metrics-source)
     - [Backfill](#backfill)
-        - [Reindex From Snapshot](#reindex-from-snapshot)
-        - [OpenSearch Ingestion](#opensearch-ingestion)
+      - [Reindex From Snapshot](#reindex-from-snapshot)
+      - [OpenSearch Ingestion](#opensearch-ingestion)
     - [Snapshot](#snapshot)
     - [Metadata Migration](#metadata-migration)
     - [Replay](#replay)
     - [Kafka](#kafka)
     - [Client Options](#client-options)
-- [Usage](#usage)
+  - [Usage](#usage)
     - [Library](#library)
     - [CLI](#cli)
-        - [Global Options](#global-options)
-        - [Objects](#objects)
-        - [Commands \& options](#commands--options)
-- [Development](#development)
+      - [Global Options](#global-options)
+      - [Objects](#objects)
+      - [Commands \& options](#commands--options)
+  - [Development](#development)
     - [Unit Tests](#unit-tests)
     - [Coverage](#coverage)
 
@@ -135,7 +136,7 @@ Most of the parameters for these two are the same, with some additional ones spe
 - `reindex_from_snapshot`
     - `snapshot_repo`: optional, path to the snapshot repo. If not provided, ???
     - `snapshot_name`: optional, name of the snapshot to use as the source. If not provided, ???
-    - `scale`: optional int, number of instances to enable when `backfill start` is run. While running, this is modifiable with `backfill scale X`. Default is 1.
+    - `scale`: optional int, number of instances to enable when `backfill start` is run. While running, this is modifiable with `backfill scale X`. Default is 5.
 
 There is also a block that specifies the deployment type. Exactly one of the following blocks must be present:
 

diff --git a/...ion/src/main/docker/migrationConsole/lib/console_link/console_link/models/backfill_rfs.py b/...ion/src/main/docker/migrationConsole/lib/console_link/console_link/models/backfill_rfs.py
@@ -92,7 +92,7 @@ def __init__(self, config: Dict, target_cluster: Cluster, client_options: Option
         super().__init__(config)
         self.client_options = client_options
         self.target_cluster = target_cluster
-        self.default_scale = self.config["reindex_from_snapshot"].get("scale", 1)
+        self.default_scale = self.config["reindex_from_snapshot"].get("scale", 5)
 
         self.ecs_config = self.config["reindex_from_snapshot"]["ecs"]
         self.ecs_client = ECSService(cluster_name=self.ecs_config["cluster_name"],

diff --git a/...e/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill.py b/...e/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill.py
@@ -173,16 +173,16 @@ def test_cant_instantiate_with_multiple_rfs_deployment_types():
 
 
 def test_ecs_rfs_backfill_start_sets_ecs_desired_count(ecs_rfs_backfill, mocker):
-    assert ecs_rfs_backfill.default_scale == 1
+    assert ecs_rfs_backfill.default_scale == 5
     mock = mocker.patch.object(ECSService, 'set_desired_count', autospec=True)
     ecs_rfs_backfill.start()
 
     assert isinstance(ecs_rfs_backfill, ECSRFSBackfill)
-    mock.assert_called_once_with(ecs_rfs_backfill.ecs_client, 1)
+    mock.assert_called_once_with(ecs_rfs_backfill.ecs_client, 5)
 
 
 def test_ecs_rfs_backfill_stop_sets_ecs_desired_count(ecs_rfs_backfill, mocker):
-    assert ecs_rfs_backfill.default_scale == 1
+    assert ecs_rfs_backfill.default_scale == 5
     mock = mocker.patch.object(ECSService, 'set_desired_count', autospec=True)
     ecs_rfs_backfill.stop()
 

diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts
@@ -40,6 +40,7 @@ export class ECSService {
 
 export class RFSBackfillYaml {
     ecs: ECSService;
+    scale: number = 5;
     constructor() {
         this.ecs = new ECSService();
     }