From ff16d1a337410912181fa7bcbd04e39538edc518 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 11:28:42 +0100 Subject: [PATCH 01/13] Add restore command inside kourou --- features/Elasticsearch.feature | 6 +++++ src/commands/es/snapshot/restore.ts | 42 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 src/commands/es/snapshot/restore.ts diff --git a/features/Elasticsearch.feature b/features/Elasticsearch.feature index 8d99763f..d040e3ac 100644 --- a/features/Elasticsearch.feature +++ b/features/Elasticsearch.feature @@ -57,6 +57,12 @@ Feature: Elasticsearch commands | arg | backup | | Then I should match stdout with "test-snapshot" + Scenario: Restore ES data from a snapshot + When I run the command "es:snapshot:restore" with: + | arg | backup | | + | arg | test-snapshot | | + Then I should match stdout with "Success" + Scenario: Dump and restore ES data to a dump folder using the pattern option Given an index "nyc-open-data" Given a collection "nyc-open-data":"yellow-taxi" diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts new file mode 100644 index 00000000..7316f28b --- /dev/null +++ b/src/commands/es/snapshot/restore.ts @@ -0,0 +1,42 @@ +import { flags } from "@oclif/command"; +import { Client } from "@elastic/elasticsearch"; + +import { Kommand } from "../../../common"; + +export default class EsSnapshotsRestore extends Kommand { + static initSdk = false; + + static description = "Restore a snapshot repository inside an ES instance"; + + static flags = { + node: flags.string({ + char: "n", + description: "Elasticsearch server URL", + default: "http://localhost:9200", + }), + help: flags.help(), + }; + + static args = [ + { name: "repository", description: "ES repository name", required: true }, + { name: "name", description: "ES snapshot name", required: true }, + ]; + + async runSafe() { + const esClient = new Client({ node: this.flags.node }); + + const esRequest = { + repository: this.args.repository, + snapshot: this.args.name, + body: { + feature_states: ["geoip"], + include_global_state: false, + indices: "*", + }, + }; + + const response = await esClient.snapshot.create(esRequest); + + this.logOk(`Success ${JSON.stringify(response.body)}`); + } +} From a014174e838512eef5b2e91be74c678941766df1 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 11:49:35 +0100 Subject: [PATCH 02/13] Silly me, use the restore function --- src/commands/es/snapshot/restore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index 7316f28b..a02ca9d5 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -35,7 +35,7 @@ export default class EsSnapshotsRestore extends Kommand { }, }; - const response = await esClient.snapshot.create(esRequest); + const response = await esClient.snapshot.restore(esRequest); this.logOk(`Success ${JSON.stringify(response.body)}`); } From f9f4eaf2dee41d7bffdfb8b5b1a15d4e56afbe9a Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 11:58:01 +0100 Subject: [PATCH 03/13] Modify body query for the restore --- src/commands/es/snapshot/restore.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index a02ca9d5..d5b26cfc 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -29,7 +29,6 @@ export default class EsSnapshotsRestore extends Kommand { repository: this.args.repository, snapshot: this.args.name, body: { - feature_states: ["geoip"], include_global_state: false, indices: "*", }, From 399f679d2f82ed390482717ed40c0c1fe96caba5 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 12:06:44 +0100 Subject: [PATCH 04/13] Close indices before restoring --- src/commands/es/snapshot/restore.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index d5b26cfc..7e331e7a 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -25,6 +25,10 @@ export default class EsSnapshotsRestore extends Kommand { async runSafe() { const esClient = new Client({ node: this.flags.node }); + await esClient.indices.close({ + index: "*", + }); + const esRequest = { repository: this.args.repository, snapshot: this.args.name, From 401e41916e842936337b496b91c58024b15a4e7a Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 12:12:50 +0100 Subject: [PATCH 05/13] feature_state to none --- src/commands/es/snapshot/restore.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index 7e331e7a..7e1749e3 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -33,6 +33,7 @@ export default class EsSnapshotsRestore extends Kommand { repository: this.args.repository, snapshot: this.args.name, body: { + feature_states: ["none"], include_global_state: false, indices: "*", }, From 94106e80ac2e97a8aaa975b3a99b847f7795057d Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 12:25:26 +0100 Subject: [PATCH 06/13] indices selected --- src/commands/es/snapshot/restore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index 7e1749e3..293bfff8 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -35,7 +35,7 @@ export default class EsSnapshotsRestore extends Kommand { body: { feature_states: ["none"], include_global_state: false, - indices: "*", + indices: "-.ds-.logs-deprecation.elasticsearch-default, *", }, }; From 27e5956e8181458060b439bc43a293e247df31dc Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 13:45:41 +0100 Subject: [PATCH 07/13] Use expend wildcards --- src/commands/es/snapshot/restore.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index 293bfff8..956986ec 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -27,6 +27,7 @@ export default class EsSnapshotsRestore extends Kommand { await esClient.indices.close({ index: "*", + expand_wildcards: "all", }); const esRequest = { From f5d2b0d09e58f8b55d1ffb72366e5e751c6e2e28 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 13:53:31 +0100 Subject: [PATCH 08/13] Changes for create request --- src/commands/es/snapshot/create.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/commands/es/snapshot/create.ts b/src/commands/es/snapshot/create.ts index 6041abd1..ef9c635f 100644 --- a/src/commands/es/snapshot/create.ts +++ b/src/commands/es/snapshot/create.ts @@ -28,7 +28,11 @@ export default class EsSnapshotsCreate extends Kommand { const esRequest = { repository: this.args.repository, snapshot: this.args.name, - body: {}, + body: { + indices: "*", + include_global_state: false, + partial: false, + }, }; const response = await esClient.snapshot.create(esRequest); From 668b21f8607b0b738d1f481d09131a8acd128c89 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 13:58:40 +0100 Subject: [PATCH 09/13] indices selected --- src/commands/es/snapshot/restore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/es/snapshot/restore.ts b/src/commands/es/snapshot/restore.ts index 956986ec..749e49b3 100644 --- a/src/commands/es/snapshot/restore.ts +++ b/src/commands/es/snapshot/restore.ts @@ -36,7 +36,7 @@ export default class EsSnapshotsRestore extends Kommand { body: { feature_states: ["none"], include_global_state: false, - indices: "-.ds-.logs-deprecation.elasticsearch-default, *", + indices: "*", }, }; From 0da35edc13232a0c1a799a1754738b2c361874df Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 14:04:04 +0100 Subject: [PATCH 10/13] Move migrate before a restore, because of shard being destroyed by restore --- features/Elasticsearch.feature | 52 +++++++++++++++++----------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/features/Elasticsearch.feature b/features/Elasticsearch.feature index d040e3ac..f689aa06 100644 --- a/features/Elasticsearch.feature +++ b/features/Elasticsearch.feature @@ -28,6 +28,32 @@ Feature: Elasticsearch commands Then I should match stdout with "{"index": "%kuzzle.users", "alias": "@%kuzzle.users"}" @mappings + Scenario: Dump and restore ES data to a dump folder using the pattern option + Given an index "nyc-open-data" + Given a collection "nyc-open-data":"yellow-taxi" + Then I create the following document: + | _id | "chuon-chuon-kim" | + | body | { "city": "hcmc", "district": 1 } | + Then I create the following document: + | _id | "the-hive-vn" | + | body | { "city": "hcmc", "district": 2 } | + Then I create the following document: + | _id | "the-hive-th" | + | body | { "city": "changmai", "district": 7 } | + Then I count 3 documents + Then I run the command "es:migrate" with: + | flag | --src | http://localhost:9200 | + | flag | --dest | ./kourou-dump | + Then I should have 3 lines in file "./kourou-dump/&nyc-open-data.yellow-taxi.json" + Then I run the command "es:migrate" with: + | flag | --src | ./kourou-dump | + | flag | --dest | http://localhost:9200 | + | flag | --reset | | + | flag | --no-interactive | | + Given an existing collection "nyc-open-data":"yellow-taxi" + Then I refresh the collection + And I count 3 documents + Scenario: Insert ES document Given a collection "nyc-open-data":"green-taxi" When I run the command "es:indices:insert" with: @@ -62,29 +88,3 @@ Feature: Elasticsearch commands | arg | backup | | | arg | test-snapshot | | Then I should match stdout with "Success" - - Scenario: Dump and restore ES data to a dump folder using the pattern option - Given an index "nyc-open-data" - Given a collection "nyc-open-data":"yellow-taxi" - Then I create the following document: - | _id | "chuon-chuon-kim" | - | body | { "city": "hcmc", "district": 1 } | - Then I create the following document: - | _id | "the-hive-vn" | - | body | { "city": "hcmc", "district": 2 } | - Then I create the following document: - | _id | "the-hive-th" | - | body | { "city": "changmai", "district": 7 } | - Then I count 3 documents - Then I run the command "es:migrate" with: - | flag | --src | http://localhost:9200 | - | flag | --dest | ./kourou-dump | - Then I should have 3 lines in file "./kourou-dump/&nyc-open-data.yellow-taxi.json" - Then I run the command "es:migrate" with: - | flag | --src | ./kourou-dump | - | flag | --dest | http://localhost:9200 | - | flag | --reset | | - | flag | --no-interactive | | - Given an existing collection "nyc-open-data":"yellow-taxi" - Then I refresh the collection - And I count 3 documents From 7d01dd80ee3157369efdbd5e13617701325601e9 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 14:20:01 +0100 Subject: [PATCH 11/13] Elasticsearch tests --- features/Elasticsearch.feature | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/features/Elasticsearch.feature b/features/Elasticsearch.feature index f689aa06..d684c325 100644 --- a/features/Elasticsearch.feature +++ b/features/Elasticsearch.feature @@ -29,7 +29,6 @@ Feature: Elasticsearch commands @mappings Scenario: Dump and restore ES data to a dump folder using the pattern option - Given an index "nyc-open-data" Given a collection "nyc-open-data":"yellow-taxi" Then I create the following document: | _id | "chuon-chuon-kim" | @@ -55,14 +54,14 @@ Feature: Elasticsearch commands And I count 3 documents Scenario: Insert ES document - Given a collection "nyc-open-data":"green-taxi" + Given a collection "nyc-open-data":"blue-taxi" When I run the command "es:indices:insert" with: - | arg | &nyc-open-data.yellow-taxi | | - | flag | --id | kindred | - | flag | --body | {} | + | arg | &nyc-open-data.blue-taxi | | + | flag | --id | kindred | + | flag | --body | {} | When I run the command "es:indices:get" with args: - | "&nyc-open-data.yellow-taxi" | - | "kindred" | + | "&nyc-open-data.blue-taxi" | + | "kindred" | Then I should match stdout with "kindred" Scenario: Create a snapshot repository From 72e313125acd100b935b0e9b8f4463b0ffb3e789 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 14:28:23 +0100 Subject: [PATCH 12/13] Launch Elasticsearch feature last, because we destroy shards when restoring a snapshot --- features/{Elasticsearch.feature => Z_Elasticsearch.feature} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename features/{Elasticsearch.feature => Z_Elasticsearch.feature} (100%) diff --git a/features/Elasticsearch.feature b/features/Z_Elasticsearch.feature similarity index 100% rename from features/Elasticsearch.feature rename to features/Z_Elasticsearch.feature From 551782896ad1926aaf54e33b8c49a3af1f817110 Mon Sep 17 00:00:00 2001 From: rolljee Date: Wed, 24 Jan 2024 14:58:52 +0100 Subject: [PATCH 13/13] Release 0.28.1 --- README.md | 20 +++++++++++++++++++- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 80208219..0465a26f 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ $ npm install -g kourou $ kourou COMMAND running command... $ kourou (-v|--version|version) -kourou/0.28.0 darwin-arm64 node-v20.10.0 +kourou/0.28.1 darwin-arm64 node-v20.10.0 $ kourou --help [COMMAND] USAGE $ kourou COMMAND @@ -149,6 +149,7 @@ All other arguments and options will be passed as-is to the `sdk:query` method. * [`kourou es:snapshot:create REPOSITORY NAME`](#kourou-essnapshotcreate-repository-name) * [`kourou es:snapshot:create-repository REPOSITORY LOCATION`](#kourou-essnapshotcreate-repository-repository-location) * [`kourou es:snapshot:list REPOSITORY`](#kourou-essnapshotlist-repository) +* [`kourou es:snapshot:restore REPOSITORY NAME`](#kourou-essnapshotrestore-repository-name) * [`kourou file:decrypt FILE`](#kourou-filedecrypt-file) * [`kourou file:encrypt FILE`](#kourou-fileencrypt-file) * [`kourou file:test FILE`](#kourou-filetest-file) @@ -795,6 +796,23 @@ OPTIONS _See code: [lib/commands/es/snapshot/list.js](lib/commands/es/snapshot/list.js)_ +## `kourou es:snapshot:restore REPOSITORY NAME` + +Restore a snapshot repository inside an ES instance + +``` +USAGE + $ kourou es:snapshot:restore REPOSITORY NAME + +ARGUMENTS + REPOSITORY ES repository name + NAME ES snapshot name + +OPTIONS + -n, --node=node [default: http://localhost:9200] Elasticsearch server URL + --help show CLI help +``` + ## `kourou file:decrypt FILE` Decrypts an encrypted file. diff --git a/package-lock.json b/package-lock.json index f66df273..bb8b0065 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "kourou", - "version": "0.28.0", + "version": "0.28.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "kourou", - "version": "0.28.0", + "version": "0.28.1", "license": "Apache-2.0", "dependencies": { "@elastic/elasticsearch": "^7.12.0", diff --git a/package.json b/package.json index 82963ad3..a2eb9e89 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "kourou", "description": "The CLI that helps you manage your Kuzzle instances", - "version": "0.28.0", + "version": "0.28.1", "author": "The Kuzzle Team ", "bin": { "kourou": "./bin/run"