diff --git a/build.gradle b/build.gradle
index d24b3b1f..c365eea1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -220,13 +220,15 @@ asciidoctor {
'source-highlighter': 'coderay'
}
-checkstyleMain.source = 'src/main/java'
+//checkstyleMain.source = 'src/main/java'
// we use @Ignore
-forbiddenApisTest.enabled = false
-forbiddenPatterns.exclude('**/*.txt', '**/*.brk', '**/*.fst', '**/*.tree', '**/*.nrm')
-licenseHeaders.enabled = false
-dependencyLicenses.enabled = false
-thirdPartyAudit.enabled = false
-loggerUsageCheck.enabled = false
-testingConventions.enabled = false
+//forbiddenApisTest.enabled = false
+//tasks.named("forbiddenPatterns").configure {
+// exclude('**/*.txt', '**/*.brk', '**/*.fst', '**/*.tree', '**/*.nrm')
+//}
+//licenseHeaders.enabled = false
+//dependencyLicenses.enabled = false
+//thirdPartyAudit.enabled = false
+//loggerUsageCheck.enabled = false
+//testingConventions.enabled = false
validateNebulaPom.enabled = false
diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml
index 52fe33cc..9217bfcf 100644
--- a/config/checkstyle/checkstyle.xml
+++ b/config/checkstyle/checkstyle.xml
@@ -38,6 +38,22 @@ page at http://checkstyle.sourceforge.net/config.html -->
Only allows a package-info.java, not package.html. -->
+
+
+
+
+
+
+
+
+
+
@@ -73,12 +89,12 @@ page at http://checkstyle.sourceforge.net/config.html -->
-
+
-
-
-
+
+
+
@@ -184,22 +200,6 @@ page at http://checkstyle.sourceforge.net/config.html -->
-->
-
-
-
-
-
-
-
-
-
-
diff --git a/gradle.properties b/gradle.properties
index 80a756fe..8381d578 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -1,17 +1,16 @@
group = org.xbib.elasticsearch
name = elasticsearch-plugin-bundle
-version = 7.10.1.0
+version = 7.11.1.0
-elasticsearch.version = 7.10.1
+elasticsearch.version = 7.11.1
lucene.version = 8.7.0
icu4j.version = 67.1
standardnumber.version = 1.0.1
-jackson.version = 2.9.10
+jackson.version = 2.10.4
spatial4j.version = 0.7
jts.version = 1.15.1
-jna.version = 4.5.1
-checkstyle.version = 8.13
+jna.version = 5.5.0
wagon.version = 3.0.0
log4j.version = 2.13.1
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index bca17f36..12d38de6 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java
index 2320f51b..8970ffcb 100644
--- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java
+++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java
@@ -31,13 +31,13 @@
import org.xbib.elasticsearch.plugin.bundle.action.isbnformat.TransportISBNFormatAction;
import org.xbib.elasticsearch.plugin.bundle.action.langdetect.LangdetectAction;
import org.xbib.elasticsearch.plugin.bundle.action.langdetect.TransportLangdetectAction;
+import org.xbib.elasticsearch.plugin.bundle.common.standardnumber.StandardnumberService;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.autophrase.AutoPhrasingTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.baseform.BaseformTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.concat.ConcatTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.concat.PairTokenFilterFactory;
-import org.xbib.elasticsearch.plugin.bundle.index.analysis.decompound.patricia.DecompoundTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.decompound.fst.FstDecompoundTokenFilterFactory;
-import org.xbib.elasticsearch.plugin.bundle.index.analysis.lemmatize.LemmatizeTokenFilterFactory;
+import org.xbib.elasticsearch.plugin.bundle.index.analysis.decompound.patricia.DecompoundTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.german.GermanNormalizationFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.hyphen.HyphenAnalyzerProvider;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.hyphen.HyphenTokenFilterFactory;
@@ -51,6 +51,7 @@
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuNumberFormatTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuTransformTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.segmentation.IcuTokenizerFactory;
+import org.xbib.elasticsearch.plugin.bundle.index.analysis.lemmatize.LemmatizeTokenFilterFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.naturalsort.NaturalSortKeyAnalyzerProvider;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.naturalsort.NaturalSortKeyTokenizerFactory;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.sortform.SortformAnalyzerProvider;
@@ -65,12 +66,8 @@
import org.xbib.elasticsearch.plugin.bundle.index.mapper.langdetect.LangdetectMapper;
import org.xbib.elasticsearch.plugin.bundle.index.mapper.reference.ReferenceMapper;
import org.xbib.elasticsearch.plugin.bundle.index.mapper.reference.ReferenceMapperModule;
-import org.xbib.elasticsearch.plugin.bundle.index.mapper.reference.ReferenceMapperTypeParser;
-import org.xbib.elasticsearch.plugin.bundle.common.reference.ReferenceService;
import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapper;
import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapperModule;
-import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapperTypeParser;
-import org.xbib.elasticsearch.plugin.bundle.common.standardnumber.StandardnumberService;
import org.xbib.elasticsearch.plugin.bundle.query.decompound.ExactPhraseQueryBuilder;
import org.xbib.elasticsearch.plugin.bundle.rest.action.isbnformat.RestISBNFormatterAction;
import org.xbib.elasticsearch.plugin.bundle.rest.action.langdetect.RestLangdetectAction;
@@ -90,11 +87,9 @@
*/
public class BundlePlugin extends Plugin implements AnalysisPlugin, MapperPlugin, SearchPlugin, ActionPlugin {
- private static final StandardnumberMapperTypeParser standardNumberTypeParser =
- new StandardnumberMapperTypeParser();
+ private final StandardnumberMapper.TypeParser standardNumberTypeParser = StandardnumberMapper.newTypeParser();
- private static final ReferenceMapperTypeParser referenceMapperTypeParser =
- new ReferenceMapperTypeParser();
+ private final ReferenceMapper.TypeParser referenceMapperTypeParser = ReferenceMapper.PARSER;
private final Settings settings;
@@ -244,10 +239,10 @@ public Map getMappers() {
extra.put(ReferenceMapper.CONTENT_TYPE, referenceMapperTypeParser);
}
if (settings.getAsBoolean("plugins.xbib.langdetect.enabled", true)) {
- extra.put(LangdetectMapper.CONTENT_TYPE, new LangdetectMapper.TypeParser());
+ extra.put(LangdetectMapper.CONTENT_TYPE, LangdetectMapper.PARSER);
}
if (settings.getAsBoolean("plugins.xbib.icu.enabled", true)) {
- extra.put(IcuCollationKeyFieldMapper.CONTENT_TYPE, new IcuCollationKeyFieldMapper.TypeParser());
+ extra.put(IcuCollationKeyFieldMapper.CONTENT_TYPE, IcuCollationKeyFieldMapper.PARSER);
}
return extra;
}
@@ -304,9 +299,6 @@ public Collection createGuiceModules() {
@Override
public Collection> getGuiceServiceClasses() {
Collection> extra = new ArrayList<>();
- if (settings.getAsBoolean("plugins.xbib.reference.enabled", true)) {
- extra.add(ReferenceService.class);
- }
if (settings.getAsBoolean("plugins.xbib.standardnumber.enabled", true)) {
extra.add(StandardnumberService.class);
}
diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java
index 99a74dc9..03a8d10f 100644
--- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java
+++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java
@@ -23,7 +23,7 @@ public class TransportISBNFormatAction extends TransportAction DEFAULT_STANDARD_NUMBERS =
Arrays.asList("isbn", "issn", "ismn", "isni", "orcid", "ppn", "zdb");
- /**
- * Called from StandardnumberTokenFilterFactory.
- * @param standardNumberTypeParser the type parser
- */
- public void setStandardNumberTypeParser(StandardnumberMapper.TypeParser standardNumberTypeParser) {
+ @Inject
+ public StandardnumberService(StandardnumberMapper.TypeParser standardNumberTypeParser) {
standardNumberTypeParser.setService(this);
}
diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java
index 42b2cf59..304f3b4f 100644
--- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java
+++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java
@@ -5,8 +5,8 @@
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.AbstractTokenFilterFactory;
-import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapper;
import org.xbib.elasticsearch.plugin.bundle.common.standardnumber.StandardnumberService;
+import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapper;
/**
* Standard number token filter factory.
@@ -24,8 +24,7 @@ public StandardnumberTokenFilterFactory(IndexSettings indexSettings,
StandardnumberMapper.TypeParser standardNumberTypeParser) {
super(indexSettings, name, settings);
this.settings = settings;
- this.standardnumberService = new StandardnumberService();
- this.standardnumberService.setStandardNumberTypeParser(standardNumberTypeParser);
+ this.standardnumberService = new StandardnumberService(standardNumberTypeParser);
}
@Override
diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java
index f370a103..c206086c 100644
--- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java
+++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java
@@ -2,6 +2,8 @@
import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RawCollationKey;
+import com.ibm.icu.text.RuleBasedCollator;
+import com.ibm.icu.util.ULocale;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedSetDocValuesField;
@@ -14,40 +16,33 @@
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.lucene.Lucene;
-import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
-import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
+import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
-import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.SourceValueFetcher;
import org.elasticsearch.index.mapper.StringFieldType;
+import org.elasticsearch.index.mapper.TextParams;
import org.elasticsearch.index.mapper.TextSearchInfo;
-import org.elasticsearch.index.mapper.TypeParsers;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
import org.elasticsearch.search.lookup.SearchLookup;
-import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuCollationKeyAnalyzerProvider;
import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IndexableBinaryStringTools;
import java.io.IOException;
import java.time.ZoneId;
+import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import java.util.function.LongSupplier;
import java.util.function.Supplier;
/**
@@ -57,35 +52,49 @@ public class IcuCollationKeyFieldMapper extends FieldMapper {
public static final String CONTENT_TYPE = "icu_collation_key";
- public static final FieldType FIELD_TYPE = new FieldType();
-
- public static class Defaults {
-
- static {
- FIELD_TYPE.setTokenized(false);
- FIELD_TYPE.setOmitNorms(true);
- FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
- FIELD_TYPE.freeze();
- }
- }
-
public static final class CollationFieldType extends StringFieldType {
private final Collator collator;
+ private final String nullValue;
+ private final int ignoreAbove;
- public CollationFieldType(String name, boolean isSearchable, boolean hasDocValues, Collator collator, Map meta) {
- super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
- setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+ public CollationFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues,
+ Collator collator, String nullValue, int ignoreAbove, Map meta) {
+ super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
this.collator = collator;
+ this.nullValue = nullValue;
+ this.ignoreAbove = ignoreAbove;
+ }
+
+ public CollationFieldType(String name, boolean searchable, Collator collator) {
+ this(name, searchable, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap());
}
public CollationFieldType(String name, Collator collator) {
- this(name, true, true, collator, Collections.emptyMap());
+ this(name, true, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap());
+ }
+
+ @Override
+ public String typeName() {
+ return CONTENT_TYPE;
}
@Override
- public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
- throw new UnsupportedOperationException();
+ public ValueFetcher valueFetcher(QueryShardContext context, String format) {
+ if (format != null) {
+ throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+ }
+
+ return new SourceValueFetcher(name(), context, nullValue) {
+ @Override
+ protected String parseSourceValue(Object value) {
+ String keywordValue = value.toString();
+ if (keywordValue.length() > ignoreAbove) {
+ return null;
+ }
+ return keywordValue;
+ }
+ };
}
@Override
@@ -110,11 +119,6 @@ public boolean equals(Object o)
return collator != null ? collator.equals(that.collator) : that.collator == null;
}
- @Override
- public String typeName() {
- return CONTENT_TYPE;
- }
-
public Collator collator() {
return collator;
}
@@ -153,24 +157,27 @@ protected BytesRef indexedValueForSearch(Object value) {
@Override
public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions,
boolean transpositions, QueryShardContext context) {
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException("[fuzzy] queries are not supported on [" + CONTENT_TYPE + "] fields.");
}
@Override
public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) {
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException("[prefix] queries are not supported on [" + CONTENT_TYPE + "] fields.");
}
@Override
public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates,
MultiTermQuery.RewriteMethod method, QueryShardContext context) {
- throw new UnsupportedOperationException();
+ throw new UnsupportedOperationException("[regexp] queries are not supported on [" + CONTENT_TYPE + "] fields.");
}
+ /*
@Override
- public DocValueFormat docValueFormat(final String format, ZoneId timeZone) {
- return COLLATE_FORMAT;
+ public Query wildcardQuery(
+ String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) {
+ throw new UnsupportedOperationException("[wildcard] queries are not supported on [" + CONTENT_TYPE + "] fields.");
}
+ */
public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() {
@Override
@@ -182,16 +189,6 @@ public String getWriteableName() {
public void writeTo(StreamOutput out) {
}
- @Override
- public String format(long value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String format(double value) {
- throw new UnsupportedOperationException();
- }
-
@Override
public String format(BytesRef value) {
int encodedLength = IndexableBinaryStringTools.getEncodedLength(value.bytes, value.offset, value.length);
@@ -200,16 +197,6 @@ public String format(BytesRef value) {
return new String(encoded, 0, encodedLength);
}
- @Override
- public long parseLong(String value, boolean roundUp, LongSupplier now) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public double parseDouble(String value, boolean roundUp, LongSupplier now) {
- throw new UnsupportedOperationException();
- }
-
@Override
public BytesRef parseBytesRef(String value) {
char[] encoded = value.toCharArray();
@@ -219,217 +206,260 @@ public BytesRef parseBytesRef(String value) {
return new BytesRef(decoded);
}
};
- }
-
- public static class Builder extends FieldMapper.Builder {
- private Settings.Builder settingsBuilder;
- protected String nullValue;
-
- public Builder(String name) {
- super(name, FIELD_TYPE);
- builder = this;
- this.settingsBuilder = Settings.builder();
- }
@Override
- public Builder indexOptions(IndexOptions indexOptions) {
- if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) {
- throw new IllegalArgumentException("The [" + CONTENT_TYPE + "] field does not support positions, got [index_options]="
- + indexOptionToString(indexOptions));
- }
- return super.indexOptions(indexOptions);
- }
-
- public String rules() {
- return settingsBuilder.get("rules");
- }
-
- public Builder rules(final String rules) {
- settingsBuilder.put("rules", rules);
- return this;
- }
-
- public String language() {
- return settingsBuilder.get("language");
- }
-
- public Builder language(final String language) {
- settingsBuilder.put("language", language);
- return this;
- }
-
- public String country() {
- return settingsBuilder.get("country");
- }
-
- public Builder country(final String country) {
- settingsBuilder.put("country", country);
- return this;
+ public DocValueFormat docValueFormat(final String format, final ZoneId timeZone) {
+ return COLLATE_FORMAT;
}
- public String variant() {
- return settingsBuilder.get("variant");
- }
+ }
- public Builder variant(final String variant) {
- settingsBuilder.put("variant", variant);
- return this;
- }
+ private static IcuCollationKeyFieldMapper toType(FieldMapper in) {
+ return (IcuCollationKeyFieldMapper) in;
+ }
+
+ public static class Builder extends FieldMapper.Builder {
+
+ final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true);
+ final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
+ final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false);
+
+ final Parameter indexOptions
+ = Parameter.restrictedStringParam("index_options", false, m -> toType(m).indexOptions, "docs", "freqs");
+ final Parameter hasNorms = TextParams.norms(false, m -> !toType(m).fieldType.omitNorms());
+
+ final Parameter> meta = Parameter.metaParam();
+
+ final Parameter rules
+ = Parameter.stringParam("rules", false, m -> toType(m).params.rules, null).acceptsNull();
+ final Parameter language
+ = Parameter.stringParam("language", false, m -> toType(m).params.language, null).acceptsNull();
+ final Parameter country
+ = Parameter.stringParam("country", false, m -> toType(m).params.country, null).acceptsNull();
+ final Parameter variant
+ = Parameter.stringParam("variant", false, m -> toType(m).params.variant, null).acceptsNull();
+ final Parameter strength
+ = Parameter.stringParam("strength", false, m -> toType(m).params.strength, null).acceptsNull();
+ final Parameter decomposition
+ = Parameter.stringParam("decomposition", false, m -> toType(m).params.decomposition, null)
+ .acceptsNull();
+ final Parameter alternate
+ = Parameter.stringParam("alternate", false, m -> toType(m).params.alternate, null).acceptsNull();
+ final Parameter caseLevel = Parameter.boolParam("case_level", false, m -> toType(m).params.caseLevel, false);
+ final Parameter caseFirst
+ = Parameter.stringParam("case_first", false, m -> toType(m).params.caseFirst, null).acceptsNull();
+ final Parameter numeric = Parameter.boolParam("numeric", false, m -> toType(m).params.numeric, false);
+ final Parameter variableTop
+ = Parameter.stringParam("variable_top", false, m -> toType(m).params.variableTop, null).acceptsNull();
+ final Parameter hiraganaQuaternaryMode
+ = Parameter.boolParam("hiragana_quaternary_mode", false, m -> toType(m).params.hiraganaQuaternaryMode, false).acceptsNull();
+
+
+ final Parameter ignoreAbove
+ = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE)
+ .setValidator(v -> {
+ if (v < 0) {
+ throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
+ }
+ });
+ final Parameter nullValue
+ = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull();
- public String strength() {
- return settingsBuilder.get("strength");
+ public Builder(String name) {
+ super(name);
}
- public Builder strength(final String strength) {
- settingsBuilder.put("strength", strength);
+ Builder nullValue(String nullValue) {
+ this.nullValue.setValue(nullValue);
return this;
}
- public String decomposition() {
- return settingsBuilder.get("decomposition");
- }
-
- public Builder decomposition(final String decomposition) {
- settingsBuilder.put("decomposition", decomposition);
+ Builder ignoreAbove(int ignoreAbove) {
+ this.ignoreAbove.setValue(ignoreAbove);
return this;
}
- public String alternate() {
- return settingsBuilder.get("alternate");
+ @Override
+ protected List> getParameters() {
+ return Arrays.asList(indexed, hasDocValues, stored, indexOptions, hasNorms,
+ rules, language, country, variant, strength, decomposition, alternate,
+ caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode,
+ ignoreAbove, nullValue, meta);
+ }
+
+ private CollatorParams collatorParams() {
+ CollatorParams params = new CollatorParams();
+ params.rules = rules.getValue();
+ params.language = language.getValue();
+ params.country = country.getValue();
+ params.variant = variant.getValue();
+ params.strength = strength.getValue();
+ params.decomposition = decomposition.getValue();
+ params.alternate = alternate.getValue();
+ params.caseLevel = caseLevel.getValue();
+ params.caseFirst = caseFirst.getValue();
+ params.numeric = numeric.getValue();
+ params.variableTop = variableTop.getValue();
+ params.hiraganaQuaternaryMode = hiraganaQuaternaryMode.getValue();
+ return params;
+ }
+
+ private FieldType buildFieldType() {
+ FieldType ft = new FieldType();
+ ft.setTokenized(false);
+ ft.setOmitNorms(hasNorms.getValue() == false);
+ ft.setIndexOptions(TextParams.toIndexOptions(indexed.getValue(), indexOptions.getValue()));
+ ft.setStored(stored.getValue());
+ return ft;
}
-
- public Builder alternate(final String alternate) {
- settingsBuilder.put("alternate", alternate);
- return this;
+ @Override
+ public IcuCollationKeyFieldMapper build(ContentPath contentPath) {
+ final CollatorParams params = collatorParams();
+ final Collator collator = params.buildCollator();
+ CollationFieldType ft = new CollationFieldType(buildFullName(contentPath), indexed.getValue(),
+ stored.getValue(), hasDocValues.getValue(), collator, nullValue.getValue(), ignoreAbove.getValue(),
+ meta.getValue());
+ return new IcuCollationKeyFieldMapper(name, buildFieldType(), ft,
+ multiFieldsBuilder.build(this, contentPath), copyTo.build(), collator, this);
}
+ }
- public boolean caseLevel() {
- return settingsBuilder.get("case_level") != null && Boolean.parseBoolean(settingsBuilder.get("case_level"));
- }
+ public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+
+ private static class CollatorParams {
+ private String rules;
+ private String language;
+ private String country;
+ private String variant;
+ private String strength;
+ private String decomposition;
+ private String alternate;
+ private boolean caseLevel;
+ private String caseFirst;
+ private boolean numeric;
+ private String variableTop;
+ private boolean hiraganaQuaternaryMode;
- public Builder caseLevel(final boolean caseLevel) {
- settingsBuilder.put("case_level", caseLevel);
- return this;
- }
+ public Collator buildCollator() {
+ Collator collator;
+ if (rules != null) {
+ try {
+ collator = new RuleBasedCollator(rules);
+ } catch (Exception e) {
+ throw new IllegalArgumentException("Failed to parse collation rules", e);
+ }
+ } else {
+ if (language != null) {
+ ULocale locale;
+ if (country != null) {
+ if (variant != null) {
+ locale = new ULocale(language, country, variant);
+ } else {
+ locale = new ULocale(language, country);
+ }
+ } else {
+ locale = new ULocale(language);
+ }
+ collator = Collator.getInstance(locale);
+ } else {
+ collator = Collator.getInstance(ULocale.ROOT);
+ }
+ }
- public String caseFirst() {
- return settingsBuilder.get("case_first");
- }
+ // set the strength flag, otherwise it will be the default.
+ if (strength != null) {
+ if (strength.equalsIgnoreCase("primary")) {
+ collator.setStrength(Collator.PRIMARY);
+ } else if (strength.equalsIgnoreCase("secondary")) {
+ collator.setStrength(Collator.SECONDARY);
+ } else if (strength.equalsIgnoreCase("tertiary")) {
+ collator.setStrength(Collator.TERTIARY);
+ } else if (strength.equalsIgnoreCase("quaternary")) {
+ collator.setStrength(Collator.QUATERNARY);
+ } else if (strength.equalsIgnoreCase("identical")) {
+ collator.setStrength(Collator.IDENTICAL);
+ } else {
+ throw new IllegalArgumentException("Invalid strength: " + strength);
+ }
+ }
- public Builder caseFirst(final String caseFirst) {
- settingsBuilder.put("case_first", caseFirst);
- return this;
- }
+ // set the decomposition flag, otherwise it will be the default.
+ if (decomposition != null) {
+ if (decomposition.equalsIgnoreCase("no")) {
+ collator.setDecomposition(Collator.NO_DECOMPOSITION);
+ } else if (decomposition.equalsIgnoreCase("canonical")) {
+ collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION);
+ } else {
+ throw new IllegalArgumentException("Invalid decomposition: " + decomposition);
+ }
+ }
- public boolean numeric() {
- return settingsBuilder.get("numeric") != null && Boolean.parseBoolean(settingsBuilder.get("numeric"));
- }
+ // expert options: concrete subclasses are always a RuleBasedCollator
+ RuleBasedCollator rbc = (RuleBasedCollator) collator;
+ if (alternate != null) {
+ if (alternate.equalsIgnoreCase("shifted")) {
+ rbc.setAlternateHandlingShifted(true);
+ } else if (alternate.equalsIgnoreCase("non-ignorable")) {
+ rbc.setAlternateHandlingShifted(false);
+ } else {
+ throw new IllegalArgumentException("Invalid alternate: " + alternate);
+ }
+ }
- public Builder numeric(final boolean numeric) {
- settingsBuilder.put("numeric", numeric);
- return this;
- }
+ if (caseLevel) {
+ rbc.setCaseLevel(true);
+ }
- public Builder nullValue(String nullValue) {
- this.nullValue = nullValue;
- return this;
- }
+ if (caseFirst != null) {
+ if (caseFirst.equalsIgnoreCase("lower")) {
+ rbc.setLowerCaseFirst(true);
+ } else if (caseFirst.equalsIgnoreCase("upper")) {
+ rbc.setUpperCaseFirst(true);
+ } else {
+ throw new IllegalArgumentException("Invalid caseFirst: " + caseFirst);
+ }
+ }
- public Collator buildCollator() {
- return IcuCollationKeyAnalyzerProvider.createCollator(settingsBuilder.build());
- }
+ if (numeric) {
+ rbc.setNumericCollation(true);
+ }
- @Override
- public IcuCollationKeyFieldMapper build(BuilderContext context) {
- final Collator collator = buildCollator();
- CollationFieldType ft = new CollationFieldType(buildFullName(context), indexed, hasDocValues, collator, meta);
- return new IcuCollationKeyFieldMapper(name, fieldType, ft,
- multiFieldsBuilder.build(this, context), copyTo, settingsBuilder.build(), collator, nullValue);
- }
- }
+ if (variableTop != null) {
+ rbc.setVariableTop(variableTop);
+ }
- public static class TypeParser implements Mapper.TypeParser {
- @Override
- public Mapper.Builder parse(String name, Map node, ParserContext parserContext)
- throws MapperParsingException {
- Builder builder = new Builder(name);
- TypeParsers.parseField(builder, name, node, parserContext);
- for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext(); ) {
- Map.Entry entry = iterator.next();
- String fieldName = entry.getKey();
- Object fieldNode = entry.getValue();
- switch (fieldName) {
- case "null_value":
- if (fieldNode == null) {
- throw new MapperParsingException("property [null_value] cannot be null");
- }
- builder.nullValue(fieldNode.toString());
- iterator.remove();
- break;
- case "norms":
- builder.omitNorms(!XContentMapValues.nodeBooleanValue(fieldNode));
- iterator.remove();
- break;
- case "rules":
- builder.rules(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "language":
- builder.language(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "country":
- builder.country(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "variant":
- builder.variant(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "strength":
- builder.strength(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "decomposition":
- builder.decomposition(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "alternate":
- builder.alternate(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "case_level":
- builder.caseLevel(XContentMapValues.nodeBooleanValue(fieldNode));
- iterator.remove();
- break;
- case "case_first":
- builder.caseFirst(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "numeric":
- builder.numeric(XContentMapValues.nodeBooleanValue(fieldNode));
- iterator.remove();
- break;
- default:
- break;
- }
+ if (hiraganaQuaternaryMode) {
+ rbc.setHiraganaQuaternary(true);
}
- return builder;
+
+ // freeze so thread-safe
+ return collator.freeze();
}
}
- private final Settings collatorSettings;
+ private final int ignoreAbove;
private final Collator collator;
+ private final CollatorParams params;
private final String nullValue;
-
- protected IcuCollationKeyFieldMapper(String simpleName, FieldType fieldType, MappedFieldType defaultFieldType,
- MultiFields multiFields,
- CopyTo copyTo, Settings collatorSettings, Collator collator, String nullValue) {
- super(simpleName, fieldType, defaultFieldType, multiFields, copyTo);
+ private final FieldType fieldType;
+ private final boolean indexed;
+ private final boolean hasDocValues;
+ private final String indexOptions;
+
+ protected IcuCollationKeyFieldMapper(String simpleName, FieldType fieldType,
+ MappedFieldType mappedFieldType,
+ MultiFields multiFields, CopyTo copyTo,
+ Collator collator, Builder builder) {
+ super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
assert collator.isFrozen();
- this.collatorSettings = collatorSettings;
+ this.fieldType = fieldType;
+ this.params = builder.collatorParams();
+ this.ignoreAbove = builder.ignoreAbove.getValue();
this.collator = collator;
- this.nullValue = nullValue;
+ this.nullValue = builder.nullValue.getValue();
+ this.indexed = builder.indexed.getValue();
+ this.hasDocValues = builder.hasDocValues.getValue();
+ this.indexOptions = builder.indexOptions.getValue();
}
@Override
@@ -443,86 +473,8 @@ protected String contentType() {
}
@Override
- protected void mergeOptions(FieldMapper other, List conflicts) {
- IcuCollationKeyFieldMapper icuMergeWith = (IcuCollationKeyFieldMapper) other;
- if (!Objects.equals(collator, icuMergeWith.collator)) {
- conflicts.add("mapper [" + name() + "] has different [collator]");
- }
- if (!Objects.equals(collatorSettings.get("rules"), icuMergeWith.collatorSettings.get("rules"))) {
- conflicts.add("Cannot update rules setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("language"), icuMergeWith.collatorSettings.get("language"))) {
- conflicts.add("Cannot update language setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("country"), icuMergeWith.collatorSettings.get("country"))) {
- conflicts.add("Cannot update country setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("variant"), icuMergeWith.collatorSettings.get("variant"))) {
- conflicts.add("Cannot update variant setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("strength"), icuMergeWith.collatorSettings.get("strength"))) {
- conflicts.add("Cannot update strength setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("decomposition"), icuMergeWith.collatorSettings.get("decomposition"))) {
- conflicts.add("Cannot update decomposition setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("alternate"), icuMergeWith.collatorSettings.get("alternate"))) {
- conflicts.add("Cannot update alternate setting for [" + CONTENT_TYPE + "]");
- }
- if (collatorSettings.getAsBoolean("case_level", true) != icuMergeWith.collatorSettings.getAsBoolean("case_level", true)) {
- conflicts.add("Cannot update case_level setting for [" + CONTENT_TYPE + "]");
- }
- if (!Objects.equals(collatorSettings.get("case_first"), icuMergeWith.collatorSettings.get("case_first"))) {
- conflicts.add("Cannot update case_first setting for [" + CONTENT_TYPE + "]");
- }
- if (collatorSettings.getAsBoolean("numeric", true) != icuMergeWith.collatorSettings.getAsBoolean("numeric", true)) {
- conflicts.add("Cannot update numeric setting for [" + CONTENT_TYPE + "]");
- }
- }
-
-
- @Override
- protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
- super.doXContentBody(builder, includeDefaults, params);
- if (fieldType.indexOptions() != IndexOptions.NONE && (includeDefaults || fieldType.indexOptions() != IndexOptions.DOCS)) {
- builder.field("index_options", indexOptionToString(fieldType.indexOptions()));
- }
- if (nullValue != null) {
- builder.field("null_value", nullValue);
- }
- if (includeDefaults || fieldType.omitNorms() != KeywordFieldMapper.Defaults.FIELD_TYPE.omitNorms()) {
- builder.field("norms", fieldType.omitNorms() == false);
- }
- if (includeDefaults) {
- builder.field("rules", collatorSettings.get("rules"));
- }
- if (includeDefaults) {
- builder.field("language", collatorSettings.get("language"));
- }
- if (includeDefaults) {
- builder.field("country", collatorSettings.get("country"));
- }
- if (includeDefaults) {
- builder.field("variant", collatorSettings.get("variant"));
- }
- if (includeDefaults) {
- builder.field("strength", collatorSettings.get("strength"));
- }
- if (includeDefaults) {
- builder.field("decomposition", collatorSettings.get("decomposition"));
- }
- if (includeDefaults) {
- builder.field("alternate", collatorSettings.get("alternate"));
- }
- if (includeDefaults) {
- builder.field("case_level", collatorSettings.getAsBoolean("case_level", false));
- }
- if (includeDefaults) {
- builder.field("case_first", collatorSettings.get("case_first"));
- }
- if (includeDefaults) {
- builder.field("numeric", collatorSettings.getAsBoolean("numeric", false));
- }
+ public FieldMapper.Builder getMergeBuilder() {
+ return new Builder(simpleName()).init(this);
}
@Override
@@ -538,16 +490,20 @@ protected void parseCreateField(ParseContext context) throws IOException {
value = parser.textOrNull();
}
}
- if (value == null) {
+
+ if (value == null || value.length() > ignoreAbove) {
return;
}
+
RawCollationKey key = collator.getRawCollationKey(value, null);
final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size);
+
if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
Field field = new Field(mappedFieldType.name(), binaryValue, fieldType);
context.doc().add(field);
}
- if (fieldType().hasDocValues()) {
+
+ if (hasDocValues) {
context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
} else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
createFieldNamesField(context);
diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java
index 5207e34c..3e73b290 100644
--- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java
+++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java
@@ -16,22 +16,23 @@
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.SimpleMappedFieldType;
import org.elasticsearch.index.mapper.TextSearchInfo;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.lookup.SearchLookup;
import org.xbib.elasticsearch.plugin.bundle.common.langdetect.LangdetectService;
import org.xbib.elasticsearch.plugin.bundle.common.langdetect.Language;
import org.xbib.elasticsearch.plugin.bundle.common.langdetect.LanguageDetectionException;
@@ -40,10 +41,11 @@
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.function.Function;
/**
* Language detection field mapper.
@@ -52,33 +54,65 @@ public class LangdetectMapper extends FieldMapper {
public static final String CONTENT_TYPE = "langdetect";
- public static final FieldType FIELD_TYPE = new FieldType();
-
private static final Logger logger = LogManager.getLogger(LangdetectMapper.class);
- static {
- FIELD_TYPE.setStored(true);
- FIELD_TYPE.setOmitNorms(true);
- FIELD_TYPE.setDocValuesType(DocValuesType.NONE);
- FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
- FIELD_TYPE.freeze();
+ final FieldType fieldType;
+
+ public static class Defaults {
+ public static final FieldType FIELD_TYPE = new FieldType();
+ public static final int NTRIALS = 7;
+ public static final double ALPHA = 0.5;
+ public static final double ALPHA_WIDTH = 0.05;
+ public static final int ITERATION_LIMIT = 10_000;
+ public static final double PROB_THRESHOLD = 0.1;
+ public static final double CONV_THRESHOLD = 0.99999;
+ public static final int BASE_FREQ = 10_000;
+
+ static {
+ FIELD_TYPE.setStored(true);
+ FIELD_TYPE.setOmitNorms(true);
+ FIELD_TYPE.setDocValuesType(DocValuesType.NONE);
+ FIELD_TYPE.setIndexOptions(IndexOptions.DOCS);
+ FIELD_TYPE.freeze();
+ }
}
- private final LangdetectService langdetectService;
+ private static LangdetectMapper toType(FieldMapper in) {
+ return (LangdetectMapper) in;
+ }
+
+ private static Builder builder(FieldMapper in) {
+ return toType(in).builder;
+ }
- private final LanguageTo languageTo;
- private ParseContext context;
+ private final Builder builder;
+
+ private LangdetectService langdetectService = null;
+ private LanguageTo languageTo = null;
public LangdetectMapper(String simpleName,
FieldType fieldType,
MappedFieldType mappedFieldType,
MultiFields multiFields,
CopyTo copyTo,
- LanguageTo languageTo,
- LangdetectService langdetectService) {
- super(simpleName, fieldType, mappedFieldType, multiFields, copyTo);
- this.langdetectService = langdetectService;
- this.languageTo = languageTo;
+ Builder builder) {
+ super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo);
+ this.fieldType = fieldType;
+ this.builder = builder;
+ }
+
+ public LangdetectService getLangdetectService() {
+ return Objects.requireNonNullElseGet(langdetectService, () -> {
+ langdetectService = new LangdetectService(builder.settingsBuilder.build());
+ return langdetectService;
+ });
+ }
+
+ public LanguageTo getLanguageTo() {
+ return Objects.requireNonNullElseGet(languageTo, () -> {
+ languageTo = LanguageTo.builder().add(builder.languageToMap.getValue()).build();
+ return languageTo;
+ });
}
@Override
@@ -87,7 +121,8 @@ protected String contentType() {
}
@Override
- protected void mergeOptions(FieldMapper other, List conflicts) {
+ public FieldMapper.Builder getMergeBuilder() {
+ return new LangdetectMapper.Builder(simpleName(), builder.indexAnalyzers).init(this);
}
@Override
@@ -102,7 +137,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
if (value == null) {
return;
}
- boolean isBinary = langdetectService.getSettings().getAsBoolean("binary", false);
+ boolean isBinary = getLangdetectService().getSettings().getAsBoolean("binary", false);
if (isBinary) {
try {
byte[] b = parser.binaryValue();
@@ -125,12 +160,12 @@ protected void parseCreateField(ParseContext context) throws IOException {
}
}
try {
- List langs = langdetectService.detectAll(value);
+ List langs = getLangdetectService().detectAll(value);
for (Language lang : langs) {
Field field = new Field(fieldType().name(), lang.getLanguage(), fieldType);
context.doc().add(field);
- if (languageTo.languageToFields().containsKey(lang.getLanguage())) {
- parseLanguageToFields(context, languageTo.languageToFields().get(lang.getLanguage()));
+ if (getLanguageTo().languageToFields().containsKey(lang.getLanguage())) {
+ parseLanguageToFields(context, getLanguageTo().languageToFields().get(lang.getLanguage()));
}
}
} catch (LanguageDetectionException e) {
@@ -139,42 +174,17 @@ protected void parseCreateField(ParseContext context) throws IOException {
}
}
- private static Iterable extractFieldNames(final String fullPath) {
- return () -> new Iterator<>() {
-
- int endIndex = nextEndIndex(0);
-
- private int nextEndIndex(int index) {
- while (index < fullPath.length() && fullPath.charAt(index) != '.') {
- index += 1;
- }
- return index;
- }
-
- @Override
- public boolean hasNext() {
- return endIndex <= fullPath.length();
- }
-
- @Override
- public String next() {
- final String result = fullPath.substring(0, endIndex);
- endIndex = nextEndIndex(endIndex + 1);
- return result;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
- };
- }
-
@Override
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
- super.doXContentBody(builder, includeDefaults, params);
- langdetectService.getSettings().toXContent(builder, params);
- languageTo.toXContent(builder, params);
+ builder.field("type", contentType());
+ Builder mergeBuilder = new Builder(simpleName(), this.builder.indexAnalyzers);
+ mergeBuilder.init(this);
+ mergeBuilder.toXContent(builder, includeDefaults);
+ multiFields.toXContent(builder, params);
+ if (null != copyTo) {
+ copyTo.toXContent(builder, params);
+ }
+ getLanguageTo().toXContent(builder, params);
}
@SuppressWarnings("unchecked")
@@ -209,119 +219,216 @@ private static void parseLanguageToFields(ParseContext originalContext, Object l
}
}
- public static class Builder extends FieldMapper.Builder {
-
- protected int positionIncrementGap = -1;
-
- protected LanguageTo languageTo = LanguageTo.builder().build();
+ public static class Builder extends FieldMapper.Builder {
protected Settings.Builder settingsBuilder = Settings.builder();
- public Builder(String name) {
- super(name, FIELD_TYPE);
- this.builder = this;
+ private final Parameter store = Parameter.storeParam(
+ m -> builder(m).store.getValue(),
+ false
+ );
+
+ final Parameter nTrial = Parameter.intParam(
+ "number_of_trials",
+ true,
+ m -> {
+ Parameter nTrial = builder(m).nTrial;
+ Integer value = nTrial.getValue();
+ builder(m).settingsBuilder.put(nTrial.name, value);
+ return value;
+ },
+ Defaults.NTRIALS
+ );
+ private final Parameter alpha = Parameter.doubleParam(
+ "alpha",
+ true,
+ m -> {
+ Parameter alpha = builder(m).alpha;
+ Double value = alpha.getValue();
+ builder(m).settingsBuilder.put(alpha.name, value);
+ return value;
+ },
+ Defaults.ALPHA
+ );
+ private final Parameter alphaWidth = Parameter.doubleParam(
+ "alpha_width",
+ true,
+ m -> {
+ Parameter alphaWidth = builder(m).alphaWidth;
+ Double value = alphaWidth.getValue();
+ builder(m).settingsBuilder.put(alphaWidth.name, value);
+ return value;
+ },
+ Defaults.ALPHA_WIDTH
+ );
+ private final Parameter iterationLimit = Parameter.intParam(
+ "iteration_limit",
+ true,
+ m -> {
+ Parameter iterationLimit = builder(m).iterationLimit;
+ Integer value = iterationLimit.getValue();
+ builder(m).settingsBuilder.put(iterationLimit.name, value);
+ return value;
+ },
+ Defaults.ITERATION_LIMIT
+ );
+ private final Parameter probThreshold = Parameter.doubleParam(
+ "prob_threshold",
+ true,
+ m -> {
+ Parameter probThreshold = builder(m).probThreshold;
+ Double value = probThreshold.getValue();
+ builder(m).settingsBuilder.put(probThreshold.name, value);
+ return value;
+ },
+ Defaults.PROB_THRESHOLD
+ );
+ private final Parameter convThreshold = Parameter.doubleParam(
+ "conv_threshold",
+ true,
+ m -> {
+ Parameter convThreshold = builder(m).convThreshold;
+ Double value = convThreshold.getValue();
+ builder(m).settingsBuilder.put(convThreshold.name, value);
+ return value;
+ },
+ Defaults.CONV_THRESHOLD
+ );
+ private final Parameter baseFreq = Parameter.intParam(
+ "base_freq",
+ true,
+ m -> {
+ Parameter baseFreq = builder(m).baseFreq;
+ Integer value = baseFreq.getValue();
+ builder(m).settingsBuilder.put(baseFreq.name, value);
+ return value;
+ },
+ Defaults.BASE_FREQ
+ );
+ private final Parameter filterPattern = Parameter.stringParam(
+ "pattern",
+ true,
+ m -> {
+ Parameter filterPattern = builder(m).filterPattern;
+ String value = filterPattern.getValue();
+ builder(m).settingsBuilder.put(filterPattern.name, value);
+ return value;
+ },
+ null
+ );
+ private final Parameter max = Parameter.intParam(
+ "max",
+ true,
+ m -> {
+ Parameter max = builder(m).max;
+ Integer value = max.getValue();
+ builder(m).settingsBuilder.put(max.name, value);
+ return value;
+ },
+ Integer.MAX_VALUE
+ );
+ private final Parameter profile = Parameter.stringParam(
+ "profile",
+ true,
+ m -> {
+ Parameter profile = builder(m).profile;
+ String value = profile.getValue();
+ builder(m).settingsBuilder.put(profile.name, value);
+ return value;
+ },
+ null
+ );
+ private final Parameter binary = Parameter.boolParam(
+ "binary",
+ true,
+ m -> {
+ Parameter binary = builder(m).binary;
+ Boolean value = binary.getValue();
+ builder(m).settingsBuilder.put(binary.name, value);
+ return value;
+ },
+ false
+ );
+
+ private final Parameter> map = mapParam(
+ "map",
+ true,
+ m -> {
+ Map map = builder(m).map.getValue();
+ for (Map.Entry entry : map.entrySet()) {
+ builder(m).settingsBuilder.put("map." + entry.getKey(), (String) entry.getValue());
+ }
+ return map;
+ },
+ Collections.emptyMap()
+ );
+
+ private final Parameter> languages = listParam(
+ "languages",
+ true,
+ m -> {
+ List value = builder(m).languages.getValue();
+ settingsBuilder.putList("languages", value);
+ return value;
+ },
+ Collections.emptyList()
+ );
+
+ private final Parameter> languageToMap = mapParam(
+ "language_to",
+ true,
+ m -> builder(m).languageToMap.getValue(),
+ Collections.emptyMap()
+ ).neverSerialize();
+
+ private final Parameter positionIncrementGap = Parameter.intParam(
+ "position_increment_gap",
+ true,
+ m -> builder(m).positionIncrementGap.getValue(),
+ -1
+ );
+
+ final IndexAnalyzers indexAnalyzers;
+ final NamedAnalyzer indexAnalyzer;
+ final NamedAnalyzer searchAnalyzer;
+ final NamedAnalyzer searchQuoteAnalyzer;
+
+ public Builder(String name, IndexAnalyzers indexAnalyzers) {
+ super(name);
+ this.indexAnalyzers = indexAnalyzers;
+ this.indexAnalyzer = indexAnalyzers.getDefaultIndexAnalyzer();
+ this.searchAnalyzer = indexAnalyzers.getDefaultSearchAnalyzer();
+ this.searchQuoteAnalyzer = indexAnalyzers.getDefaultSearchQuoteAnalyzer();
+ }
+
+ @Override
+ protected List> getParameters() {
+ return Arrays.asList(store, positionIncrementGap,
+ nTrial, alpha, alphaWidth, iterationLimit, probThreshold, convThreshold,
+ baseFreq, filterPattern, max, binary, map, languages, profile, languageToMap);
}
public NamedAnalyzer indexAnalyzer() {
- return indexAnalyzer;
+ return indexAnalyzer();
}
public NamedAnalyzer searchAnalyzer() {
- return searchAnalyzer;
+ return searchAnalyzer();
}
public NamedAnalyzer searchQuoteAnalyzer() {
- return searchQuoteAnalyzer;
- }
-
- public Builder positionIncrementGap(int positionIncrementGap) {
- this.positionIncrementGap = positionIncrementGap;
- return this;
- }
-
- public Builder ntrials(int trials) {
- settingsBuilder.put("number_of_trials", trials);
- return this;
- }
-
- public Builder alpha(double alpha) {
- settingsBuilder.put("alpha", alpha);
- return this;
- }
-
- public Builder alphaWidth(double alphaWidth) {
- settingsBuilder.put("alpha_width", alphaWidth);
- return this;
- }
-
- public Builder iterationLimit(int iterationLimit) {
- settingsBuilder.put("iteration_limit", iterationLimit);
- return this;
- }
-
- public Builder probThreshold(double probThreshold) {
- settingsBuilder.put("prob_threshold", probThreshold);
- return this;
- }
-
- public Builder convThreshold(double convThreshold) {
- settingsBuilder.put("conv_threshold", convThreshold);
- return this;
- }
-
- public Builder baseFreq(int baseFreq) {
- settingsBuilder.put("base_freq", baseFreq);
- return this;
- }
-
- public Builder pattern(String pattern) {
- settingsBuilder.put("pattern", pattern);
- return this;
- }
-
- public Builder max(int max) {
- settingsBuilder.put("max", max);
- return this;
- }
-
- public Builder binary(boolean binary) {
- settingsBuilder.put("binary", binary);
- return this;
- }
-
- public Builder map(Map map) {
- for (Map.Entry entry : map.entrySet()) {
- settingsBuilder.put("map." + entry.getKey(), (String) entry.getValue());
- }
- return this;
- }
-
- public Builder languages(String[] languages) {
- List list = Arrays.asList(languages);
- settingsBuilder.putList("languages", list);
- return this;
- }
-
- public Builder profile(String profile) {
- settingsBuilder.put("profile", profile);
- return this;
- }
-
- public Builder languageTo(LanguageTo languageTo) {
- this.languageTo = languageTo;
- return this;
+ return searchQuoteAnalyzer();
}
@Override
- public LangdetectMapper build(BuilderContext context) {
- LangdetectService service = new LangdetectService(settingsBuilder.build());
+ public LangdetectMapper build(ContentPath context) {
return new LangdetectMapper(name,
- fieldType,
+ Defaults.FIELD_TYPE,
new LangdetectFieldType(buildFullName(context)),
multiFieldsBuilder.build(this, context),
- copyTo,
- languageTo,
- service);
+ copyTo.build(),
+ this
+ );
}
}
@@ -329,7 +436,6 @@ public static class LangdetectFieldType extends SimpleMappedFieldType {
public LangdetectFieldType(String name) {
super(name, true, false, true, TextSearchInfo.NONE, Collections.emptyMap());
- setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
}
/** Returns the indexed value used to construct search "values".
@@ -340,7 +446,7 @@ protected BytesRef indexedValueForSearch(Object value) {
}
@Override
- public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
+ public ValueFetcher valueFetcher(QueryShardContext context, String format) {
throw new UnsupportedOperationException();
}
@@ -376,110 +482,10 @@ public Query termsQuery(List values, QueryShardContext context) {
}
- public static class TypeParser implements Mapper.TypeParser {
+ public static final FieldMapper.TypeParser PARSER
+ = new FieldMapper.TypeParser((n, c) -> new LangdetectMapper.Builder(n, c.getIndexAnalyzers()));
- @Override
- public Mapper.Builder parse(String name, Map mapping, ParserContext parserContext) {
- Builder builder = new Builder(name);
- Iterator> iterator = mapping.entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry entry = iterator.next();
- String fieldName = entry.getKey();
- Object fieldNode = entry.getValue();
- switch (fieldName) {
- case "analyzer":
- case "include_in_all":
- iterator.remove();
- break;
- case "position_increment_gap":
- int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(fieldNode, -1);
- if (newPositionIncrementGap < 0) {
- throw new MapperParsingException("position_increment_gap less than 0 aren't allowed.");
- }
- builder.positionIncrementGap(newPositionIncrementGap);
- if (builder.indexAnalyzer() == null) {
- builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer());
- }
- if (builder.searchAnalyzer() == null) {
- builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer());
- }
- if (builder.searchQuoteAnalyzer() == null) {
- builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer());
- }
- iterator.remove();
- break;
- case "store":
- builder.store(XContentMapValues.nodeBooleanValue(fieldNode));
- iterator.remove();
- break;
- case "number_of_trials":
- builder.ntrials(XContentMapValues.nodeIntegerValue(fieldNode));
- iterator.remove();
- break;
- case "alpha":
- builder.alpha(XContentMapValues.nodeDoubleValue(fieldNode));
- iterator.remove();
- break;
- case "alpha_width":
- builder.alphaWidth(XContentMapValues.nodeDoubleValue(fieldNode));
- iterator.remove();
- break;
- case "iteration_limit":
- builder.iterationLimit(XContentMapValues.nodeIntegerValue(fieldNode));
- iterator.remove();
- break;
- case "prob_threshold":
- builder.probThreshold(XContentMapValues.nodeDoubleValue(fieldNode));
- iterator.remove();
- break;
- case "conv_threshold":
- builder.convThreshold(XContentMapValues.nodeDoubleValue(fieldNode));
- iterator.remove();
- break;
- case "base_freq":
- builder.baseFreq(XContentMapValues.nodeIntegerValue(fieldNode));
- iterator.remove();
- break;
- case "pattern":
- builder.pattern(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "max":
- builder.max(XContentMapValues.nodeIntegerValue(fieldNode));
- iterator.remove();
- break;
- case "binary":
- builder.binary(XContentMapValues.nodeBooleanValue(fieldNode));
- iterator.remove();
- break;
- case "map":
- builder.map(XContentMapValues.nodeMapValue(fieldNode, "map"));
- iterator.remove();
- break;
- case "languages":
- builder.languages(XContentMapValues.nodeStringArrayValue(fieldNode));
- iterator.remove();
- break;
- case "profile":
- builder.profile(XContentMapValues.nodeStringValue(fieldNode, null));
- iterator.remove();
- break;
- case "language_to" :
- Map map = XContentMapValues.nodeMapValue(fieldNode, null);
- LanguageTo.Builder languageToBuilder = LanguageTo.builder();
- languageToBuilder.add(map);
- builder.languageTo(languageToBuilder.build());
- iterator.remove();
- break;
- default:
- break;
- }
- }
- return builder;
- }
- }
-
- public static class LanguageTo {
+ public static class LanguageTo implements ToXContent {
private final Map languageToFields;
@@ -525,4 +531,30 @@ public Map languageToFields() {
}
}
+ public static Parameter> mapParam(
+ String name, boolean updateable,
+ Function> initializer, Map defaultValue)
+ {
+ return new Parameter<>(
+ name,
+ updateable,
+ () -> defaultValue,
+ (n, c, o) -> XContentMapValues.nodeMapValue(o, "map"),
+ initializer
+ );
+ }
+
+ public static Parameter> listParam(
+ String name, boolean updateable,
+ Function> initializer, List defaultValue)
+ {
+ return new Parameter<>(
+ name,
+ updateable,
+ () -> defaultValue,
+ (n, c, o) -> Arrays.asList(XContentMapValues.nodeStringArrayValue(o)),
+ initializer
+ );
+ }
+
}
diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java
index 9a784812..dc501e1a 100644
--- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java
+++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java
@@ -21,26 +21,28 @@
import org.elasticsearch.index.analysis.AnalyzerScope;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.FieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.SimpleMappedFieldType;
import org.elasticsearch.index.mapper.TextFieldMapper;
import org.elasticsearch.index.mapper.TextSearchInfo;
-import org.elasticsearch.index.mapper.TypeParsers;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.lookup.SearchLookup;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collections;
-import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
/**
* Reference field mapper.
@@ -48,9 +50,7 @@
public class ReferenceMapper extends FieldMapper {
private static final Logger logger = LogManager.getLogger(ReferenceMapper.class.getName());
-
public static final String CONTENT_TYPE = "ref";
-
public static final FieldType FIELD_TYPE = new FieldType();
static {
@@ -59,21 +59,15 @@ public class ReferenceMapper extends FieldMapper {
private static final CopyTo COPYTO_EMPTY = new CopyTo.Builder().build();
- private final Client client;
-
- private String index;
-
- private String type;
-
- private List fields;
-
+ private Client client;
+ private String refIndex;
+ private String refType;
+ private List refFields;
private final FieldMapper contentMapper;
-
// override copyTo in FieldMapper
private final CopyTo copyTo;
public ReferenceMapper(String simpleName,
- FieldType fieldType,
MappedFieldType defaultFieldType,
Client client,
String refindex,
@@ -81,15 +75,33 @@ public ReferenceMapper(String simpleName,
List reffields,
FieldMapper contentMapper,
MultiFields multiFields,
- CopyTo copyTo) {
- super(simpleName, fieldType, defaultFieldType, multiFields, COPYTO_EMPTY);
+ CopyTo copyTo,
+ Builder builder) {
+ super(simpleName, defaultFieldType, Lucene.STANDARD_ANALYZER, multiFields, COPYTO_EMPTY);
this.copyTo = copyTo;
this.client = client;
- this.index = refindex;
- this.type = reftype;
- this.fields = reffields;
+ this.refIndex = refindex;
+ this.refType = reftype;
+ this.refFields = reffields;
this.contentMapper = contentMapper;
+ this.builder = builder;
+ }
+
+ private static ReferenceMapper toType(FieldMapper in) {
+ return (ReferenceMapper) in;
+ }
+
+ private static ReferenceMapper.Builder builder(FieldMapper in) {
+ return toType(in).builder;
+ }
+
+ private final ReferenceMapper.Builder builder;
+
+ /*
+ public CopyTo copyTo() {
+ return this.copyTo;
}
+ */
@Override
@SuppressWarnings("unchecked")
@@ -109,25 +121,25 @@ public void parse(ParseContext originalContext) throws IOException {
if (currentFieldName != null) {
switch (currentFieldName) {
case "ref_index":
- index = parser.text();
+ refIndex = parser.text();
break;
case "ref_type":
- type = parser.text();
+ refType = parser.text();
break;
case "ref_fields":
// single field only
- fields = new LinkedList<>();
- fields.add(parser.text());
+ refFields = new LinkedList<>();
+ refFields.add(parser.text());
break;
default:
break;
}
}
} else if (token == XContentParser.Token.START_ARRAY && "ref_fields".equals(currentFieldName)) {
- fields = new LinkedList<>();
+ refFields = new LinkedList<>();
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
if (parser.text() != null) {
- fields.add(parser.text());
+ refFields.add(parser.text());
}
}
}
@@ -138,16 +150,16 @@ public void parse(ParseContext originalContext) throws IOException {
}
context = context.createExternalValueContext(content);
contentMapper.parse(context);
- if (client != null && index != null && type != null && fields != null) {
+ if (client != null && refIndex != null && refType != null && refFields != null) {
try {
GetResponse response = client.prepareGet()
- .setIndex(index)
- .setType(type)
+ .setIndex(refIndex)
+ .setType(refType)
.setId(content)
.execute()
.actionGet();
if (response != null && response.isExists()) {
- for (String field : fields) {
+ for (String field : refFields) {
Map source = response.getSource();
List