feat: add support for elastic v6 and v7

* Support Elasticsearch v6.x and v7.x - Update mongoose - Added es6 and es7 tests - Moved istanbul to nyc - Update and fix eslint issues * Keep number of shards for v7.x consistent with previous versions - Fix eslint * Put elastic v5 as default export Update readme Add v7 export
Bowery-RES · May 8, 2019 · 709dcbb · 709dcbb
1 parent 2617375
commit 709dcbb
Show file tree

Hide file tree

Showing 76 changed files with 19,131 additions and 124 deletions.
diff --git a/.eslintrc b/.eslintrc
@@ -20,6 +20,7 @@
     "prefer-rest-params": 0,
     "no-param-reassign": 0,
     "strict": 0,
+    "import/no-cycle": 0,
     "prefer-spread": 0,
     "prefer-destructuring": 0
   },

diff --git a/.gitignore b/.gitignore
@@ -2,4 +2,6 @@
 node_modules
 .idea
 coverage
+.nyc_output
 npm-debug.log
+.vscode
diff --git a/.travis.yml b/.travis.yml
@@ -1,5 +1,4 @@
 language: node_js
-
 cache:
   yarn: true
   directories:
@@ -12,27 +11,43 @@ node_js:
 addons:
   apt:
     packages:
-      - oracle-java8-set-default
+      - oracle-java9-set-default
 
 services:
   - mongodb
+  - elasticsearch
 
 env:
   - ES=2
   - ES=5
+  - ES=6
+  - ES=7
 
 before_install:
   - if [ "$ES" -eq "2" ]; then echo "DOWNLOADING V2"; curl -O https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.2.0/elasticsearch-2.2.0.deb && sudo dpkg -i --force-confnew elasticsearch-2.2.0.deb ; fi
   - if [ "$ES" -eq "5" ]; then echo "DOWNLOADING V5"; curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.0.0.deb && sudo dpkg -i --force-confnew elasticsearch-5.0.0.deb ; fi
-  - 'echo "script.inline: true" | sudo tee -a /etc/elasticsearch/elasticsearch.yml'
+  - if [ "$ES" -eq "6" ]; then echo "DOWNLOADING V6"; curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0.deb && sudo dpkg -i --force-confnew elasticsearch-6.0.0.deb ; fi
+  - if [ "$ES" -eq "7" ]; then echo "DOWNLOADING V7"; curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.0.1-amd64.deb && sudo dpkg -i --force-confnew elasticsearch-7.0.1-amd64.deb ; fi
+  - echo `pwd`
+  - sudo cat ./scripts/docker/es${ES}/elasticsearch.yml |sudo tee -a /etc/elasticsearch/elasticsearch.yml
   - sudo cat /etc/elasticsearch/elasticsearch.yml
-  - sudo service elasticsearch start
-  - sleep 5
+  - sudo chown -R elasticsearch:elasticsearch /etc/default/elasticsearch # https://discuss.elastic.co/t/permission-denied-starting-elasticsearch-7-0/179336
+  - sudo chown -R elasticsearch:elasticsearch /etc/elasticsearch
+  - sudo chown -R elasticsearch:elasticsearch /usr/share/elasticsearch
+  - if [ "$ES" -eq "5" -o "$ES" -eq "6" ];then sudo service elasticsearch start; fi
+  - if [ "$ES" -eq "2" -o "$ES" -eq "7" ];then sudo service elasticsearch restart; fi
+  - sleep 15
+  - sudo tail -n50 /var/log/elasticsearch/elasticsearch_xp.log || true # Don't die on this
   - sudo service elasticsearch status
 
 script:
   - if [ "$ES" -eq "2" ]; then echo "RUNNING V2"; npm run-script test-v2-covered ; fi
   - if [ "$ES" -eq "5" ]; then echo "RUNNING V5"; npm run-script test-v5-covered ; fi
+  - if [ "$ES" -eq "6" ]; then echo "RUNNING V6"; npm run-script test-v6-covered ; fi
+  - if [ "$ES" -eq "7" ]; then echo "RUNNING V7"; npm run-script test-v7-covered ; fi
+
+
+
 
 
 after_success:

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 [![NPM version][npm-image]][npm-url] [![Build Status][travis-image]][travis-url] [![Coverage Status][coverage-image]][coverage-url]
 
 mongoose-elasticsearch-xp is a [mongoose](http://mongoosejs.com/) plugin that can automatically index your models into [elasticsearch](http://www.elasticsearch.org/).
-This plugin is compatible with Elasticsearch version 2 and 5.
+This plugin is compatible with Elasticsearch version 2,5,6 and 7.
 
 - [Prerequisite](#prerequisite)
 - [Why this plugin?](#why-this-plugin)
@@ -35,7 +35,7 @@ This plugin is compatible with Elasticsearch version 2 and 5.
 mongoose-elasticsearch-xp requires:
 
   - mongoose 4.9.0, 5.0.0 or later
-  - elasticsearch 2.0, 5.0 or later
+  - elasticsearch 2.0, 5.0, 6.0, 7.0 or later
 
 ## Why this plugin?
 
@@ -53,13 +53,18 @@ npm install --save mongoose-elasticsearch-xp
 
 ## Important
 
-This plugin is configured to work with the latest version (5.x.y).
+This plugin is configured to work with the latest version (7.x.y).
 In order to use it with Elasticsearch 2.x.y, you need to use the `v2` version:
 
+It is **very strongly** recommended to fix your version by using the require with the elastic
+to prevent breaking changes
+
 ```javascript
 var mexp = require('mongoose-elasticsearch-xp').v2;
 ```
 
+Likewise for `.v5`, `.v6`, and `.v7`, `v5` is default for now.
+
 The examples below use the version 5 syntax.
 
 ## Limitation
@@ -573,6 +578,8 @@ User
 
 ```
 
+⚠️ For `v7` `analysis` needs to be wrapped in a `settings` object. Please refer to: `test/es7/model-mapping.js`
+
 You'll have to manage whether or not you need to create the mapping, mongoose-elasticsearch-xp will make no assumptions and simply attempt to create the mapping.
 If the mapping already exists, an Exception detailing such will be populated in the `err` argument.
 
@@ -779,7 +786,39 @@ User
   });
 ```
 
+#### Breaking changes for elastic v7.0
+
+[List](https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#breaking_70_analysis_changes)
+
+1 - Elasticsearch _Type has been removed
+2 - Elasticsearch SQL
+3 - Index lifecycle management
+4 - Standard token filter has been removed
+5 - nGram and edgeNGram token filter cannot be used on new indices
+    should be replaces by ngram or edge_ngram
+6 - Shards number on index creation is now `1` instead of `5`
+
+This library handles types fine for now but keep that in mind that they will be gone for v8.0.
 
+#### Contributing
+
+You will need a *mongodb* running locally either via docker or your own
+The tests currently write in a `test` collection.
+
+Ideally you would run: (example for v7)
+
+Your mongodb then,
+In one terminal: `npm run docker-v7`
+In another: `npm run test-v7`
+
+All the docker images load their own `elasticsearch.yml` config,
+In the case of `es7` you might need to edit the line
+```yml
+network.host: 127.0.0.1
+```
+for
+`network.host: _eth0_` 
+in order to test locally (**don't** commit this file change or it will break travis).
 
 [npm-url]: https://npmjs.org/package/mongoose-elasticsearch-xp
 [npm-image]: https://badge.fury.io/js/mongoose-elasticsearch-xp.svg

diff --git a/lib/index.js b/lib/index.js
@@ -1,12 +1,12 @@
 'use strict';
 
+const mongoose = require('mongoose');
 const generateMapping = require('./mapping').generate;
 const client = require('./client');
 const utils = require('./utils');
 const Bulker = require('./bulker');
-const mongoose = require('mongoose');
 
-module.exports = function(schema, options, version) {
+module.exports = function(schema, options, version = 5) {
   // clone main level of options (does not clone deeper)
   options = utils.highClone(options);
 
@@ -63,7 +63,7 @@ module.exports = function(schema, options, version) {
   }
 
   schema.statics.esOptions = esOptions;
-  schema.statics.esCreateMapping = createMapping;
+  schema.statics.esCreateMapping = createMappingWithVersion(version);
   schema.statics.esRefresh = refresh;
   schema.statics.esSearch = search;
   schema.statics.esSynchronize = synchronize;
@@ -87,22 +87,44 @@ module.exports.v2 = function(schema, options) {
   return module.exports(schema, options, 2);
 };
 
+module.exports.v5 = function(schema, options) {
+  return module.exports(schema, options, 5);
+};
+
+module.exports.v6 = function(schema, options) {
+  return module.exports(schema, options, 6);
+};
+
+module.exports.v7 = function(schema, options) {
+  return module.exports(schema, options, 7);
+};
+
+/**
+ * Wraps the model wrapping function with the version number
+ * @param {String} version
+ * @return {Function}
+ */
+function createMappingWithVersion(version) {
+  return function(settings, callback) {
+    return createMapping.call(this, settings, callback, version);
+  };
+}
 /**
  * Map the model on ElasticSearch
  * static function
  * @param {Object} [settings]
  * @param {Function} [callback]
+ * @param {String} [version] ElasticSearch version
  * @returns {Promise|undefined}
  */
-function createMapping(settings, callback) {
+function createMapping(settings, callback, version) {
   if (typeof settings === 'function') {
     callback = settings;
     settings = null;
   }
   const self = this;
   return utils.run(callback, (resolve, reject) => {
     const esOptions = self.esOptions();
-
     settings = settings || esOptions.mappingSettings || {};
 
     const mapping = {};
@@ -115,35 +137,48 @@ function createMapping(settings, callback) {
           return reject(err);
         }
         if (exists) {
+          const putMappingOpts = {
+            index: esOptions.index,
+            type: esOptions.type,
+            body: mapping,
+          };
+          if (version === 7) {
+            putMappingOpts.include_type_name = true;
+          }
           return esOptions.client.indices.putMapping(
-            {
-              index: esOptions.index,
-              type: esOptions.type,
-              body: mapping,
-            },
+            putMappingOpts,
             (err, result) => (err ? reject(err) : resolve(result))
           );
         }
-        return esOptions.client.indices.create(
-          {
+        const createIndexOpts = {
+          index: esOptions.index,
+          body: settings,
+        };
+        if (version === 7) {
+          // Keep shards settings like the previous versions
+          if (!createIndexOpts.body.settings) {
+            createIndexOpts.body = { settings: {} };
+          }
+          createIndexOpts.body.settings.number_of_shards =
+            settings.number_of_shards || 5;
+        }
+        return esOptions.client.indices.create(createIndexOpts, err => {
+          if (err) {
+            reject(err);
+            return;
+          }
+          const putMappingOpts = {
             index: esOptions.index,
-            body: settings,
-          },
-          err => {
-            if (err) {
-              reject(err);
-              return;
-            }
-            esOptions.client.indices.putMapping(
-              {
-                index: esOptions.index,
-                type: esOptions.type,
-                body: mapping,
-              },
-              (err, result) => (err ? reject(err) : resolve(result))
-            );
+            type: esOptions.type,
+            body: mapping,
+          };
+          if (version === 7) {
+            putMappingOpts.include_type_name = true;
           }
-        );
+          esOptions.client.indices.putMapping(putMappingOpts, (err, result) =>
+            err ? reject(err) : resolve(result)
+          );
+        });
       }
     );
   });
@@ -270,8 +305,8 @@ function search(query, options, callback) {
 
       const isObjectId = utils.getType(self.schema.paths._id) === 'objectid';
 
-      const ids = result.hits.hits.map(
-        hit => (isObjectId ? mongoose.Types.ObjectId(hit._id) : hit._id)
+      const ids = result.hits.hits.map(hit =>
+        isObjectId ? mongoose.Types.ObjectId(hit._id) : hit._id
       );
 
       if (idsOnly) {

diff --git a/lib/mapping.js b/lib/mapping.js
@@ -191,10 +191,20 @@ function getDefault(version) {
       mixed: 'object',
     };
   }
-  return {
-    objectid: 'keyword',
-    number: 'double',
-    mixed: 'object',
-    string: 'text',
-  };
+  if (version === 5 || version === 6) {
+    return {
+      objectid: 'keyword',
+      number: 'double',
+      mixed: 'object',
+      string: 'text',
+    };
+  }
+  if (version === 7) {
+    return {
+      objectid: 'keyword',
+      number: 'long',
+      mixed: 'object',
+      string: 'text',
+    };
+  }
 }