From 597b92440115fec0d6ae5cd179a7e521b51eeb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Swen=20Th=C3=BCmmler?= Date: Thu, 11 Feb 2021 18:10:40 +0100 Subject: [PATCH 1/7] update elasticsearch to 7.10.2 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 80a756f..45738ea 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,8 +1,8 @@ group = org.xbib.elasticsearch name = elasticsearch-plugin-bundle -version = 7.10.1.0 +version = 7.10.2.0 -elasticsearch.version = 7.10.1 +elasticsearch.version = 7.10.2 lucene.version = 8.7.0 icu4j.version = 67.1 From 3e5cda5193098b3bf6090e5c9904ff2a0292611e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Swen=20Th=C3=BCmmler?= Date: Thu, 18 Feb 2021 11:20:44 +0100 Subject: [PATCH 2/7] changes for elasticearch 7.11.0 --- build.gradle | 17 +- gradle.properties | 8 +- .../plugin/bundle/BundlePlugin.java | 22 +- .../isbnformat/TransportISBNFormatAction.java | 2 +- .../langdetect/TransportLangdetectAction.java | 2 +- .../common/reference/ReferenceService.java | 40 -- .../bundle/common/reference/package-info.java | 4 - .../standardnumber/StandardnumberService.java | 10 +- .../StandardnumberTokenFilterFactory.java | 5 +- .../icu/IcuCollationKeyFieldMapper.java | 592 ++++++++---------- .../mapper/langdetect/LangdetectMapper.java | 548 ++++++++-------- .../mapper/reference/ReferenceMapper.java | 276 ++++---- .../reference/ReferenceMapperModule.java | 8 +- .../reference/ReferenceMapperTypeParser.java | 7 - .../standardnumber/StandardnumberMapper.java | 40 +- .../StandardnumberMapperModule.java | 6 +- .../StandardnumberMapperTypeParser.java | 7 - .../plugin-metadata/plugin-security.policy | 2 +- .../langdetect/LangdetectMappingTests.java | 4 +- .../StandardnumberMappingTests.java | 12 +- 20 files changed, 777 insertions(+), 835 deletions(-) delete mode 100644 src/main/java/org/xbib/elasticsearch/plugin/bundle/common/reference/ReferenceService.java delete mode 100644 src/main/java/org/xbib/elasticsearch/plugin/bundle/common/reference/package-info.java delete mode 100644 src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperTypeParser.java delete mode 100644 src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperTypeParser.java diff --git a/build.gradle b/build.gradle index d24b3b1..1baf77f 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,7 @@ plugins { id "io.codearte.nexus-staging" version "0.21.1" id "org.xbib.gradle.plugin.jflex" version "1.4.0" id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" + id 'checkstyle' } apply plugin: 'java-library' @@ -222,11 +223,13 @@ asciidoctor { checkstyleMain.source = 'src/main/java' // we use @Ignore -forbiddenApisTest.enabled = false -forbiddenPatterns.exclude('**/*.txt', '**/*.brk', '**/*.fst', '**/*.tree', '**/*.nrm') -licenseHeaders.enabled = false -dependencyLicenses.enabled = false -thirdPartyAudit.enabled = false -loggerUsageCheck.enabled = false -testingConventions.enabled = false +//forbiddenApisTest.enabled = false +//tasks.named("forbiddenPatterns").configure { +// exclude('**/*.txt', '**/*.brk', '**/*.fst', '**/*.tree', '**/*.nrm') +//} +//licenseHeaders.enabled = false +//dependencyLicenses.enabled = false +//thirdPartyAudit.enabled = false +//loggerUsageCheck.enabled = false +//testingConventions.enabled = false validateNebulaPom.enabled = false diff --git a/gradle.properties b/gradle.properties index 80a756f..d15b59b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,16 +1,16 @@ group = org.xbib.elasticsearch name = elasticsearch-plugin-bundle -version = 7.10.1.0 +version = 7.11.0.0 -elasticsearch.version = 7.10.1 +elasticsearch.version = 7.11.0 lucene.version = 8.7.0 icu4j.version = 67.1 standardnumber.version = 1.0.1 -jackson.version = 2.9.10 +jackson.version = 2.10.4 spatial4j.version = 0.7 jts.version = 1.15.1 -jna.version = 4.5.1 +jna.version = 5.5.0 checkstyle.version = 8.13 wagon.version = 3.0.0 diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java index 2320f51..8970ffc 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/BundlePlugin.java @@ -31,13 +31,13 @@ import org.xbib.elasticsearch.plugin.bundle.action.isbnformat.TransportISBNFormatAction; import org.xbib.elasticsearch.plugin.bundle.action.langdetect.LangdetectAction; import org.xbib.elasticsearch.plugin.bundle.action.langdetect.TransportLangdetectAction; +import org.xbib.elasticsearch.plugin.bundle.common.standardnumber.StandardnumberService; import org.xbib.elasticsearch.plugin.bundle.index.analysis.autophrase.AutoPhrasingTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.baseform.BaseformTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.concat.ConcatTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.concat.PairTokenFilterFactory; -import org.xbib.elasticsearch.plugin.bundle.index.analysis.decompound.patricia.DecompoundTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.decompound.fst.FstDecompoundTokenFilterFactory; -import org.xbib.elasticsearch.plugin.bundle.index.analysis.lemmatize.LemmatizeTokenFilterFactory; +import org.xbib.elasticsearch.plugin.bundle.index.analysis.decompound.patricia.DecompoundTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.german.GermanNormalizationFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.hyphen.HyphenAnalyzerProvider; import org.xbib.elasticsearch.plugin.bundle.index.analysis.hyphen.HyphenTokenFilterFactory; @@ -51,6 +51,7 @@ import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuNumberFormatTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuTransformTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.segmentation.IcuTokenizerFactory; +import org.xbib.elasticsearch.plugin.bundle.index.analysis.lemmatize.LemmatizeTokenFilterFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.naturalsort.NaturalSortKeyAnalyzerProvider; import org.xbib.elasticsearch.plugin.bundle.index.analysis.naturalsort.NaturalSortKeyTokenizerFactory; import org.xbib.elasticsearch.plugin.bundle.index.analysis.sortform.SortformAnalyzerProvider; @@ -65,12 +66,8 @@ import org.xbib.elasticsearch.plugin.bundle.index.mapper.langdetect.LangdetectMapper; import org.xbib.elasticsearch.plugin.bundle.index.mapper.reference.ReferenceMapper; import org.xbib.elasticsearch.plugin.bundle.index.mapper.reference.ReferenceMapperModule; -import org.xbib.elasticsearch.plugin.bundle.index.mapper.reference.ReferenceMapperTypeParser; -import org.xbib.elasticsearch.plugin.bundle.common.reference.ReferenceService; import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapper; import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapperModule; -import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapperTypeParser; -import org.xbib.elasticsearch.plugin.bundle.common.standardnumber.StandardnumberService; import org.xbib.elasticsearch.plugin.bundle.query.decompound.ExactPhraseQueryBuilder; import org.xbib.elasticsearch.plugin.bundle.rest.action.isbnformat.RestISBNFormatterAction; import org.xbib.elasticsearch.plugin.bundle.rest.action.langdetect.RestLangdetectAction; @@ -90,11 +87,9 @@ */ public class BundlePlugin extends Plugin implements AnalysisPlugin, MapperPlugin, SearchPlugin, ActionPlugin { - private static final StandardnumberMapperTypeParser standardNumberTypeParser = - new StandardnumberMapperTypeParser(); + private final StandardnumberMapper.TypeParser standardNumberTypeParser = StandardnumberMapper.newTypeParser(); - private static final ReferenceMapperTypeParser referenceMapperTypeParser = - new ReferenceMapperTypeParser(); + private final ReferenceMapper.TypeParser referenceMapperTypeParser = ReferenceMapper.PARSER; private final Settings settings; @@ -244,10 +239,10 @@ public Map getMappers() { extra.put(ReferenceMapper.CONTENT_TYPE, referenceMapperTypeParser); } if (settings.getAsBoolean("plugins.xbib.langdetect.enabled", true)) { - extra.put(LangdetectMapper.CONTENT_TYPE, new LangdetectMapper.TypeParser()); + extra.put(LangdetectMapper.CONTENT_TYPE, LangdetectMapper.PARSER); } if (settings.getAsBoolean("plugins.xbib.icu.enabled", true)) { - extra.put(IcuCollationKeyFieldMapper.CONTENT_TYPE, new IcuCollationKeyFieldMapper.TypeParser()); + extra.put(IcuCollationKeyFieldMapper.CONTENT_TYPE, IcuCollationKeyFieldMapper.PARSER); } return extra; } @@ -304,9 +299,6 @@ public Collection createGuiceModules() { @Override public Collection> getGuiceServiceClasses() { Collection> extra = new ArrayList<>(); - if (settings.getAsBoolean("plugins.xbib.reference.enabled", true)) { - extra.add(ReferenceService.class); - } if (settings.getAsBoolean("plugins.xbib.standardnumber.enabled", true)) { extra.add(StandardnumberService.class); } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java index 99a74dc..03a8d10 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/action/isbnformat/TransportISBNFormatAction.java @@ -23,7 +23,7 @@ public class TransportISBNFormatAction extends TransportAction DEFAULT_STANDARD_NUMBERS = Arrays.asList("isbn", "issn", "ismn", "isni", "orcid", "ppn", "zdb"); - /** - * Called from StandardnumberTokenFilterFactory. - * @param standardNumberTypeParser the type parser - */ - public void setStandardNumberTypeParser(StandardnumberMapper.TypeParser standardNumberTypeParser) { + @Inject + public StandardnumberService(StandardnumberMapper.TypeParser standardNumberTypeParser) { standardNumberTypeParser.setService(this); } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java index 42b2cf5..304f3b4 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/analysis/standardnumber/StandardnumberTokenFilterFactory.java @@ -5,8 +5,8 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; -import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapper; import org.xbib.elasticsearch.plugin.bundle.common.standardnumber.StandardnumberService; +import org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber.StandardnumberMapper; /** * Standard number token filter factory. @@ -24,8 +24,7 @@ public StandardnumberTokenFilterFactory(IndexSettings indexSettings, StandardnumberMapper.TypeParser standardNumberTypeParser) { super(indexSettings, name, settings); this.settings = settings; - this.standardnumberService = new StandardnumberService(); - this.standardnumberService.setStandardNumberTypeParser(standardNumberTypeParser); + this.standardnumberService = new StandardnumberService(standardNumberTypeParser); } @Override diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java index f370a10..c206086 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/icu/IcuCollationKeyFieldMapper.java @@ -2,6 +2,8 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RawCollationKey; +import com.ibm.icu.text.RuleBasedCollator; +import com.ibm.icu.util.ULocale; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -14,40 +16,33 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; -import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; +import org.elasticsearch.index.mapper.TextParams; import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; -import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IcuCollationKeyAnalyzerProvider; import org.xbib.elasticsearch.plugin.bundle.index.analysis.icu.IndexableBinaryStringTools; import java.io.IOException; import java.time.ZoneId; +import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.function.LongSupplier; import java.util.function.Supplier; /** @@ -57,35 +52,49 @@ public class IcuCollationKeyFieldMapper extends FieldMapper { public static final String CONTENT_TYPE = "icu_collation_key"; - public static final FieldType FIELD_TYPE = new FieldType(); - - public static class Defaults { - - static { - FIELD_TYPE.setTokenized(false); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.freeze(); - } - } - public static final class CollationFieldType extends StringFieldType { private final Collator collator; + private final String nullValue; + private final int ignoreAbove; - public CollationFieldType(String name, boolean isSearchable, boolean hasDocValues, Collator collator, Map meta) { - super(name, true, false, true, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); + public CollationFieldType(String name, boolean isSearchable, boolean isStored, boolean hasDocValues, + Collator collator, String nullValue, int ignoreAbove, Map meta) { + super(name, isSearchable, isStored, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.collator = collator; + this.nullValue = nullValue; + this.ignoreAbove = ignoreAbove; + } + + public CollationFieldType(String name, boolean searchable, Collator collator) { + this(name, searchable, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); } public CollationFieldType(String name, Collator collator) { - this(name, true, true, collator, Collections.emptyMap()); + this(name, true, false, true, collator, null, Integer.MAX_VALUE, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; } @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { - throw new UnsupportedOperationException(); + public ValueFetcher valueFetcher(QueryShardContext context, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), context, nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + return keywordValue; + } + }; } @Override @@ -110,11 +119,6 @@ public boolean equals(Object o) return collator != null ? collator.equals(that.collator) : that.collator == null; } - @Override - public String typeName() { - return CONTENT_TYPE; - } - public Collator collator() { return collator; } @@ -153,24 +157,27 @@ protected BytesRef indexedValueForSearch(Object value) { @Override public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions, QueryShardContext context) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("[fuzzy] queries are not supported on [" + CONTENT_TYPE + "] fields."); } @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("[prefix] queries are not supported on [" + CONTENT_TYPE + "] fields."); } @Override public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxDeterminizedStates, MultiTermQuery.RewriteMethod method, QueryShardContext context) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("[regexp] queries are not supported on [" + CONTENT_TYPE + "] fields."); } + /* @Override - public DocValueFormat docValueFormat(final String format, ZoneId timeZone) { - return COLLATE_FORMAT; + public Query wildcardQuery( + String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { + throw new UnsupportedOperationException("[wildcard] queries are not supported on [" + CONTENT_TYPE + "] fields."); } + */ public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() { @Override @@ -182,16 +189,6 @@ public String getWriteableName() { public void writeTo(StreamOutput out) { } - @Override - public String format(long value) { - throw new UnsupportedOperationException(); - } - - @Override - public String format(double value) { - throw new UnsupportedOperationException(); - } - @Override public String format(BytesRef value) { int encodedLength = IndexableBinaryStringTools.getEncodedLength(value.bytes, value.offset, value.length); @@ -200,16 +197,6 @@ public String format(BytesRef value) { return new String(encoded, 0, encodedLength); } - @Override - public long parseLong(String value, boolean roundUp, LongSupplier now) { - throw new UnsupportedOperationException(); - } - - @Override - public double parseDouble(String value, boolean roundUp, LongSupplier now) { - throw new UnsupportedOperationException(); - } - @Override public BytesRef parseBytesRef(String value) { char[] encoded = value.toCharArray(); @@ -219,217 +206,260 @@ public BytesRef parseBytesRef(String value) { return new BytesRef(decoded); } }; - } - - public static class Builder extends FieldMapper.Builder { - private Settings.Builder settingsBuilder; - protected String nullValue; - - public Builder(String name) { - super(name, FIELD_TYPE); - builder = this; - this.settingsBuilder = Settings.builder(); - } @Override - public Builder indexOptions(IndexOptions indexOptions) { - if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) > 0) { - throw new IllegalArgumentException("The [" + CONTENT_TYPE + "] field does not support positions, got [index_options]=" - + indexOptionToString(indexOptions)); - } - return super.indexOptions(indexOptions); - } - - public String rules() { - return settingsBuilder.get("rules"); - } - - public Builder rules(final String rules) { - settingsBuilder.put("rules", rules); - return this; - } - - public String language() { - return settingsBuilder.get("language"); - } - - public Builder language(final String language) { - settingsBuilder.put("language", language); - return this; - } - - public String country() { - return settingsBuilder.get("country"); - } - - public Builder country(final String country) { - settingsBuilder.put("country", country); - return this; + public DocValueFormat docValueFormat(final String format, final ZoneId timeZone) { + return COLLATE_FORMAT; } - public String variant() { - return settingsBuilder.get("variant"); - } + } - public Builder variant(final String variant) { - settingsBuilder.put("variant", variant); - return this; - } + private static IcuCollationKeyFieldMapper toType(FieldMapper in) { + return (IcuCollationKeyFieldMapper) in; + } + + public static class Builder extends FieldMapper.Builder { + + final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); + final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + final Parameter stored = Parameter.storeParam(m -> toType(m).fieldType.stored(), false); + + final Parameter indexOptions + = Parameter.restrictedStringParam("index_options", false, m -> toType(m).indexOptions, "docs", "freqs"); + final Parameter hasNorms = TextParams.norms(false, m -> !toType(m).fieldType.omitNorms()); + + final Parameter> meta = Parameter.metaParam(); + + final Parameter rules + = Parameter.stringParam("rules", false, m -> toType(m).params.rules, null).acceptsNull(); + final Parameter language + = Parameter.stringParam("language", false, m -> toType(m).params.language, null).acceptsNull(); + final Parameter country + = Parameter.stringParam("country", false, m -> toType(m).params.country, null).acceptsNull(); + final Parameter variant + = Parameter.stringParam("variant", false, m -> toType(m).params.variant, null).acceptsNull(); + final Parameter strength + = Parameter.stringParam("strength", false, m -> toType(m).params.strength, null).acceptsNull(); + final Parameter decomposition + = Parameter.stringParam("decomposition", false, m -> toType(m).params.decomposition, null) + .acceptsNull(); + final Parameter alternate + = Parameter.stringParam("alternate", false, m -> toType(m).params.alternate, null).acceptsNull(); + final Parameter caseLevel = Parameter.boolParam("case_level", false, m -> toType(m).params.caseLevel, false); + final Parameter caseFirst + = Parameter.stringParam("case_first", false, m -> toType(m).params.caseFirst, null).acceptsNull(); + final Parameter numeric = Parameter.boolParam("numeric", false, m -> toType(m).params.numeric, false); + final Parameter variableTop + = Parameter.stringParam("variable_top", false, m -> toType(m).params.variableTop, null).acceptsNull(); + final Parameter hiraganaQuaternaryMode + = Parameter.boolParam("hiragana_quaternary_mode", false, m -> toType(m).params.hiraganaQuaternaryMode, false).acceptsNull(); + + + final Parameter ignoreAbove + = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE) + .setValidator(v -> { + if (v < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); + } + }); + final Parameter nullValue + = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); - public String strength() { - return settingsBuilder.get("strength"); + public Builder(String name) { + super(name); } - public Builder strength(final String strength) { - settingsBuilder.put("strength", strength); + Builder nullValue(String nullValue) { + this.nullValue.setValue(nullValue); return this; } - public String decomposition() { - return settingsBuilder.get("decomposition"); - } - - public Builder decomposition(final String decomposition) { - settingsBuilder.put("decomposition", decomposition); + Builder ignoreAbove(int ignoreAbove) { + this.ignoreAbove.setValue(ignoreAbove); return this; } - public String alternate() { - return settingsBuilder.get("alternate"); + @Override + protected List> getParameters() { + return Arrays.asList(indexed, hasDocValues, stored, indexOptions, hasNorms, + rules, language, country, variant, strength, decomposition, alternate, + caseLevel, caseFirst, numeric, variableTop, hiraganaQuaternaryMode, + ignoreAbove, nullValue, meta); + } + + private CollatorParams collatorParams() { + CollatorParams params = new CollatorParams(); + params.rules = rules.getValue(); + params.language = language.getValue(); + params.country = country.getValue(); + params.variant = variant.getValue(); + params.strength = strength.getValue(); + params.decomposition = decomposition.getValue(); + params.alternate = alternate.getValue(); + params.caseLevel = caseLevel.getValue(); + params.caseFirst = caseFirst.getValue(); + params.numeric = numeric.getValue(); + params.variableTop = variableTop.getValue(); + params.hiraganaQuaternaryMode = hiraganaQuaternaryMode.getValue(); + return params; + } + + private FieldType buildFieldType() { + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(hasNorms.getValue() == false); + ft.setIndexOptions(TextParams.toIndexOptions(indexed.getValue(), indexOptions.getValue())); + ft.setStored(stored.getValue()); + return ft; } - - public Builder alternate(final String alternate) { - settingsBuilder.put("alternate", alternate); - return this; + @Override + public IcuCollationKeyFieldMapper build(ContentPath contentPath) { + final CollatorParams params = collatorParams(); + final Collator collator = params.buildCollator(); + CollationFieldType ft = new CollationFieldType(buildFullName(contentPath), indexed.getValue(), + stored.getValue(), hasDocValues.getValue(), collator, nullValue.getValue(), ignoreAbove.getValue(), + meta.getValue()); + return new IcuCollationKeyFieldMapper(name, buildFieldType(), ft, + multiFieldsBuilder.build(this, contentPath), copyTo.build(), collator, this); } + } - public boolean caseLevel() { - return settingsBuilder.get("case_level") != null && Boolean.parseBoolean(settingsBuilder.get("case_level")); - } + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n)); + + private static class CollatorParams { + private String rules; + private String language; + private String country; + private String variant; + private String strength; + private String decomposition; + private String alternate; + private boolean caseLevel; + private String caseFirst; + private boolean numeric; + private String variableTop; + private boolean hiraganaQuaternaryMode; - public Builder caseLevel(final boolean caseLevel) { - settingsBuilder.put("case_level", caseLevel); - return this; - } + public Collator buildCollator() { + Collator collator; + if (rules != null) { + try { + collator = new RuleBasedCollator(rules); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse collation rules", e); + } + } else { + if (language != null) { + ULocale locale; + if (country != null) { + if (variant != null) { + locale = new ULocale(language, country, variant); + } else { + locale = new ULocale(language, country); + } + } else { + locale = new ULocale(language); + } + collator = Collator.getInstance(locale); + } else { + collator = Collator.getInstance(ULocale.ROOT); + } + } - public String caseFirst() { - return settingsBuilder.get("case_first"); - } + // set the strength flag, otherwise it will be the default. + if (strength != null) { + if (strength.equalsIgnoreCase("primary")) { + collator.setStrength(Collator.PRIMARY); + } else if (strength.equalsIgnoreCase("secondary")) { + collator.setStrength(Collator.SECONDARY); + } else if (strength.equalsIgnoreCase("tertiary")) { + collator.setStrength(Collator.TERTIARY); + } else if (strength.equalsIgnoreCase("quaternary")) { + collator.setStrength(Collator.QUATERNARY); + } else if (strength.equalsIgnoreCase("identical")) { + collator.setStrength(Collator.IDENTICAL); + } else { + throw new IllegalArgumentException("Invalid strength: " + strength); + } + } - public Builder caseFirst(final String caseFirst) { - settingsBuilder.put("case_first", caseFirst); - return this; - } + // set the decomposition flag, otherwise it will be the default. + if (decomposition != null) { + if (decomposition.equalsIgnoreCase("no")) { + collator.setDecomposition(Collator.NO_DECOMPOSITION); + } else if (decomposition.equalsIgnoreCase("canonical")) { + collator.setDecomposition(Collator.CANONICAL_DECOMPOSITION); + } else { + throw new IllegalArgumentException("Invalid decomposition: " + decomposition); + } + } - public boolean numeric() { - return settingsBuilder.get("numeric") != null && Boolean.parseBoolean(settingsBuilder.get("numeric")); - } + // expert options: concrete subclasses are always a RuleBasedCollator + RuleBasedCollator rbc = (RuleBasedCollator) collator; + if (alternate != null) { + if (alternate.equalsIgnoreCase("shifted")) { + rbc.setAlternateHandlingShifted(true); + } else if (alternate.equalsIgnoreCase("non-ignorable")) { + rbc.setAlternateHandlingShifted(false); + } else { + throw new IllegalArgumentException("Invalid alternate: " + alternate); + } + } - public Builder numeric(final boolean numeric) { - settingsBuilder.put("numeric", numeric); - return this; - } + if (caseLevel) { + rbc.setCaseLevel(true); + } - public Builder nullValue(String nullValue) { - this.nullValue = nullValue; - return this; - } + if (caseFirst != null) { + if (caseFirst.equalsIgnoreCase("lower")) { + rbc.setLowerCaseFirst(true); + } else if (caseFirst.equalsIgnoreCase("upper")) { + rbc.setUpperCaseFirst(true); + } else { + throw new IllegalArgumentException("Invalid caseFirst: " + caseFirst); + } + } - public Collator buildCollator() { - return IcuCollationKeyAnalyzerProvider.createCollator(settingsBuilder.build()); - } + if (numeric) { + rbc.setNumericCollation(true); + } - @Override - public IcuCollationKeyFieldMapper build(BuilderContext context) { - final Collator collator = buildCollator(); - CollationFieldType ft = new CollationFieldType(buildFullName(context), indexed, hasDocValues, collator, meta); - return new IcuCollationKeyFieldMapper(name, fieldType, ft, - multiFieldsBuilder.build(this, context), copyTo, settingsBuilder.build(), collator, nullValue); - } - } + if (variableTop != null) { + rbc.setVariableTop(variableTop); + } - public static class TypeParser implements Mapper.TypeParser { - @Override - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) - throws MapperParsingException { - Builder builder = new Builder(name); - TypeParsers.parseField(builder, name, node, parserContext); - for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext(); ) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - switch (fieldName) { - case "null_value": - if (fieldNode == null) { - throw new MapperParsingException("property [null_value] cannot be null"); - } - builder.nullValue(fieldNode.toString()); - iterator.remove(); - break; - case "norms": - builder.omitNorms(!XContentMapValues.nodeBooleanValue(fieldNode)); - iterator.remove(); - break; - case "rules": - builder.rules(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "language": - builder.language(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "country": - builder.country(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "variant": - builder.variant(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "strength": - builder.strength(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "decomposition": - builder.decomposition(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "alternate": - builder.alternate(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "case_level": - builder.caseLevel(XContentMapValues.nodeBooleanValue(fieldNode)); - iterator.remove(); - break; - case "case_first": - builder.caseFirst(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "numeric": - builder.numeric(XContentMapValues.nodeBooleanValue(fieldNode)); - iterator.remove(); - break; - default: - break; - } + if (hiraganaQuaternaryMode) { + rbc.setHiraganaQuaternary(true); } - return builder; + + // freeze so thread-safe + return collator.freeze(); } } - private final Settings collatorSettings; + private final int ignoreAbove; private final Collator collator; + private final CollatorParams params; private final String nullValue; - - protected IcuCollationKeyFieldMapper(String simpleName, FieldType fieldType, MappedFieldType defaultFieldType, - MultiFields multiFields, - CopyTo copyTo, Settings collatorSettings, Collator collator, String nullValue) { - super(simpleName, fieldType, defaultFieldType, multiFields, copyTo); + private final FieldType fieldType; + private final boolean indexed; + private final boolean hasDocValues; + private final String indexOptions; + + protected IcuCollationKeyFieldMapper(String simpleName, FieldType fieldType, + MappedFieldType mappedFieldType, + MultiFields multiFields, CopyTo copyTo, + Collator collator, Builder builder) { + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); assert collator.isFrozen(); - this.collatorSettings = collatorSettings; + this.fieldType = fieldType; + this.params = builder.collatorParams(); + this.ignoreAbove = builder.ignoreAbove.getValue(); this.collator = collator; - this.nullValue = nullValue; + this.nullValue = builder.nullValue.getValue(); + this.indexed = builder.indexed.getValue(); + this.hasDocValues = builder.hasDocValues.getValue(); + this.indexOptions = builder.indexOptions.getValue(); } @Override @@ -443,86 +473,8 @@ protected String contentType() { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - IcuCollationKeyFieldMapper icuMergeWith = (IcuCollationKeyFieldMapper) other; - if (!Objects.equals(collator, icuMergeWith.collator)) { - conflicts.add("mapper [" + name() + "] has different [collator]"); - } - if (!Objects.equals(collatorSettings.get("rules"), icuMergeWith.collatorSettings.get("rules"))) { - conflicts.add("Cannot update rules setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("language"), icuMergeWith.collatorSettings.get("language"))) { - conflicts.add("Cannot update language setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("country"), icuMergeWith.collatorSettings.get("country"))) { - conflicts.add("Cannot update country setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("variant"), icuMergeWith.collatorSettings.get("variant"))) { - conflicts.add("Cannot update variant setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("strength"), icuMergeWith.collatorSettings.get("strength"))) { - conflicts.add("Cannot update strength setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("decomposition"), icuMergeWith.collatorSettings.get("decomposition"))) { - conflicts.add("Cannot update decomposition setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("alternate"), icuMergeWith.collatorSettings.get("alternate"))) { - conflicts.add("Cannot update alternate setting for [" + CONTENT_TYPE + "]"); - } - if (collatorSettings.getAsBoolean("case_level", true) != icuMergeWith.collatorSettings.getAsBoolean("case_level", true)) { - conflicts.add("Cannot update case_level setting for [" + CONTENT_TYPE + "]"); - } - if (!Objects.equals(collatorSettings.get("case_first"), icuMergeWith.collatorSettings.get("case_first"))) { - conflicts.add("Cannot update case_first setting for [" + CONTENT_TYPE + "]"); - } - if (collatorSettings.getAsBoolean("numeric", true) != icuMergeWith.collatorSettings.getAsBoolean("numeric", true)) { - conflicts.add("Cannot update numeric setting for [" + CONTENT_TYPE + "]"); - } - } - - - @Override - protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - if (fieldType.indexOptions() != IndexOptions.NONE && (includeDefaults || fieldType.indexOptions() != IndexOptions.DOCS)) { - builder.field("index_options", indexOptionToString(fieldType.indexOptions())); - } - if (nullValue != null) { - builder.field("null_value", nullValue); - } - if (includeDefaults || fieldType.omitNorms() != KeywordFieldMapper.Defaults.FIELD_TYPE.omitNorms()) { - builder.field("norms", fieldType.omitNorms() == false); - } - if (includeDefaults) { - builder.field("rules", collatorSettings.get("rules")); - } - if (includeDefaults) { - builder.field("language", collatorSettings.get("language")); - } - if (includeDefaults) { - builder.field("country", collatorSettings.get("country")); - } - if (includeDefaults) { - builder.field("variant", collatorSettings.get("variant")); - } - if (includeDefaults) { - builder.field("strength", collatorSettings.get("strength")); - } - if (includeDefaults) { - builder.field("decomposition", collatorSettings.get("decomposition")); - } - if (includeDefaults) { - builder.field("alternate", collatorSettings.get("alternate")); - } - if (includeDefaults) { - builder.field("case_level", collatorSettings.getAsBoolean("case_level", false)); - } - if (includeDefaults) { - builder.field("case_first", collatorSettings.get("case_first")); - } - if (includeDefaults) { - builder.field("numeric", collatorSettings.getAsBoolean("numeric", false)); - } + public FieldMapper.Builder getMergeBuilder() { + return new Builder(simpleName()).init(this); } @Override @@ -538,16 +490,20 @@ protected void parseCreateField(ParseContext context) throws IOException { value = parser.textOrNull(); } } - if (value == null) { + + if (value == null || value.length() > ignoreAbove) { return; } + RawCollationKey key = collator.getRawCollationKey(value, null); final BytesRef binaryValue = new BytesRef(key.bytes, 0, key.size); + if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { Field field = new Field(mappedFieldType.name(), binaryValue, fieldType); context.doc().add(field); } - if (fieldType().hasDocValues()) { + + if (hasDocValues) { context.doc().add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); } else if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { createFieldNamesField(context); diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java index 5207e34..3e73b29 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/langdetect/LangdetectMapper.java @@ -16,22 +16,23 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.lookup.SearchLookup; import org.xbib.elasticsearch.plugin.bundle.common.langdetect.LangdetectService; import org.xbib.elasticsearch.plugin.bundle.common.langdetect.Language; import org.xbib.elasticsearch.plugin.bundle.common.langdetect.LanguageDetectionException; @@ -40,10 +41,11 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; /** * Language detection field mapper. @@ -52,33 +54,65 @@ public class LangdetectMapper extends FieldMapper { public static final String CONTENT_TYPE = "langdetect"; - public static final FieldType FIELD_TYPE = new FieldType(); - private static final Logger logger = LogManager.getLogger(LangdetectMapper.class); - static { - FIELD_TYPE.setStored(true); - FIELD_TYPE.setOmitNorms(true); - FIELD_TYPE.setDocValuesType(DocValuesType.NONE); - FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); - FIELD_TYPE.freeze(); + final FieldType fieldType; + + public static class Defaults { + public static final FieldType FIELD_TYPE = new FieldType(); + public static final int NTRIALS = 7; + public static final double ALPHA = 0.5; + public static final double ALPHA_WIDTH = 0.05; + public static final int ITERATION_LIMIT = 10_000; + public static final double PROB_THRESHOLD = 0.1; + public static final double CONV_THRESHOLD = 0.99999; + public static final int BASE_FREQ = 10_000; + + static { + FIELD_TYPE.setStored(true); + FIELD_TYPE.setOmitNorms(true); + FIELD_TYPE.setDocValuesType(DocValuesType.NONE); + FIELD_TYPE.setIndexOptions(IndexOptions.DOCS); + FIELD_TYPE.freeze(); + } } - private final LangdetectService langdetectService; + private static LangdetectMapper toType(FieldMapper in) { + return (LangdetectMapper) in; + } + + private static Builder builder(FieldMapper in) { + return toType(in).builder; + } - private final LanguageTo languageTo; - private ParseContext context; + private final Builder builder; + + private LangdetectService langdetectService = null; + private LanguageTo languageTo = null; public LangdetectMapper(String simpleName, FieldType fieldType, MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, - LanguageTo languageTo, - LangdetectService langdetectService) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); - this.langdetectService = langdetectService; - this.languageTo = languageTo; + Builder builder) { + super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, multiFields, copyTo); + this.fieldType = fieldType; + this.builder = builder; + } + + public LangdetectService getLangdetectService() { + return Objects.requireNonNullElseGet(langdetectService, () -> { + langdetectService = new LangdetectService(builder.settingsBuilder.build()); + return langdetectService; + }); + } + + public LanguageTo getLanguageTo() { + return Objects.requireNonNullElseGet(languageTo, () -> { + languageTo = LanguageTo.builder().add(builder.languageToMap.getValue()).build(); + return languageTo; + }); } @Override @@ -87,7 +121,8 @@ protected String contentType() { } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { + public FieldMapper.Builder getMergeBuilder() { + return new LangdetectMapper.Builder(simpleName(), builder.indexAnalyzers).init(this); } @Override @@ -102,7 +137,7 @@ protected void parseCreateField(ParseContext context) throws IOException { if (value == null) { return; } - boolean isBinary = langdetectService.getSettings().getAsBoolean("binary", false); + boolean isBinary = getLangdetectService().getSettings().getAsBoolean("binary", false); if (isBinary) { try { byte[] b = parser.binaryValue(); @@ -125,12 +160,12 @@ protected void parseCreateField(ParseContext context) throws IOException { } } try { - List langs = langdetectService.detectAll(value); + List langs = getLangdetectService().detectAll(value); for (Language lang : langs) { Field field = new Field(fieldType().name(), lang.getLanguage(), fieldType); context.doc().add(field); - if (languageTo.languageToFields().containsKey(lang.getLanguage())) { - parseLanguageToFields(context, languageTo.languageToFields().get(lang.getLanguage())); + if (getLanguageTo().languageToFields().containsKey(lang.getLanguage())) { + parseLanguageToFields(context, getLanguageTo().languageToFields().get(lang.getLanguage())); } } } catch (LanguageDetectionException e) { @@ -139,42 +174,17 @@ protected void parseCreateField(ParseContext context) throws IOException { } } - private static Iterable extractFieldNames(final String fullPath) { - return () -> new Iterator<>() { - - int endIndex = nextEndIndex(0); - - private int nextEndIndex(int index) { - while (index < fullPath.length() && fullPath.charAt(index) != '.') { - index += 1; - } - return index; - } - - @Override - public boolean hasNext() { - return endIndex <= fullPath.length(); - } - - @Override - public String next() { - final String result = fullPath.substring(0, endIndex); - endIndex = nextEndIndex(endIndex + 1); - return result; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - langdetectService.getSettings().toXContent(builder, params); - languageTo.toXContent(builder, params); + builder.field("type", contentType()); + Builder mergeBuilder = new Builder(simpleName(), this.builder.indexAnalyzers); + mergeBuilder.init(this); + mergeBuilder.toXContent(builder, includeDefaults); + multiFields.toXContent(builder, params); + if (null != copyTo) { + copyTo.toXContent(builder, params); + } + getLanguageTo().toXContent(builder, params); } @SuppressWarnings("unchecked") @@ -209,119 +219,216 @@ private static void parseLanguageToFields(ParseContext originalContext, Object l } } - public static class Builder extends FieldMapper.Builder { - - protected int positionIncrementGap = -1; - - protected LanguageTo languageTo = LanguageTo.builder().build(); + public static class Builder extends FieldMapper.Builder { protected Settings.Builder settingsBuilder = Settings.builder(); - public Builder(String name) { - super(name, FIELD_TYPE); - this.builder = this; + private final Parameter store = Parameter.storeParam( + m -> builder(m).store.getValue(), + false + ); + + final Parameter nTrial = Parameter.intParam( + "number_of_trials", + true, + m -> { + Parameter nTrial = builder(m).nTrial; + Integer value = nTrial.getValue(); + builder(m).settingsBuilder.put(nTrial.name, value); + return value; + }, + Defaults.NTRIALS + ); + private final Parameter alpha = Parameter.doubleParam( + "alpha", + true, + m -> { + Parameter alpha = builder(m).alpha; + Double value = alpha.getValue(); + builder(m).settingsBuilder.put(alpha.name, value); + return value; + }, + Defaults.ALPHA + ); + private final Parameter alphaWidth = Parameter.doubleParam( + "alpha_width", + true, + m -> { + Parameter alphaWidth = builder(m).alphaWidth; + Double value = alphaWidth.getValue(); + builder(m).settingsBuilder.put(alphaWidth.name, value); + return value; + }, + Defaults.ALPHA_WIDTH + ); + private final Parameter iterationLimit = Parameter.intParam( + "iteration_limit", + true, + m -> { + Parameter iterationLimit = builder(m).iterationLimit; + Integer value = iterationLimit.getValue(); + builder(m).settingsBuilder.put(iterationLimit.name, value); + return value; + }, + Defaults.ITERATION_LIMIT + ); + private final Parameter probThreshold = Parameter.doubleParam( + "prob_threshold", + true, + m -> { + Parameter probThreshold = builder(m).probThreshold; + Double value = probThreshold.getValue(); + builder(m).settingsBuilder.put(probThreshold.name, value); + return value; + }, + Defaults.PROB_THRESHOLD + ); + private final Parameter convThreshold = Parameter.doubleParam( + "conv_threshold", + true, + m -> { + Parameter convThreshold = builder(m).convThreshold; + Double value = convThreshold.getValue(); + builder(m).settingsBuilder.put(convThreshold.name, value); + return value; + }, + Defaults.CONV_THRESHOLD + ); + private final Parameter baseFreq = Parameter.intParam( + "base_freq", + true, + m -> { + Parameter baseFreq = builder(m).baseFreq; + Integer value = baseFreq.getValue(); + builder(m).settingsBuilder.put(baseFreq.name, value); + return value; + }, + Defaults.BASE_FREQ + ); + private final Parameter filterPattern = Parameter.stringParam( + "pattern", + true, + m -> { + Parameter filterPattern = builder(m).filterPattern; + String value = filterPattern.getValue(); + builder(m).settingsBuilder.put(filterPattern.name, value); + return value; + }, + null + ); + private final Parameter max = Parameter.intParam( + "max", + true, + m -> { + Parameter max = builder(m).max; + Integer value = max.getValue(); + builder(m).settingsBuilder.put(max.name, value); + return value; + }, + Integer.MAX_VALUE + ); + private final Parameter profile = Parameter.stringParam( + "profile", + true, + m -> { + Parameter profile = builder(m).profile; + String value = profile.getValue(); + builder(m).settingsBuilder.put(profile.name, value); + return value; + }, + null + ); + private final Parameter binary = Parameter.boolParam( + "binary", + true, + m -> { + Parameter binary = builder(m).binary; + Boolean value = binary.getValue(); + builder(m).settingsBuilder.put(binary.name, value); + return value; + }, + false + ); + + private final Parameter> map = mapParam( + "map", + true, + m -> { + Map map = builder(m).map.getValue(); + for (Map.Entry entry : map.entrySet()) { + builder(m).settingsBuilder.put("map." + entry.getKey(), (String) entry.getValue()); + } + return map; + }, + Collections.emptyMap() + ); + + private final Parameter> languages = listParam( + "languages", + true, + m -> { + List value = builder(m).languages.getValue(); + settingsBuilder.putList("languages", value); + return value; + }, + Collections.emptyList() + ); + + private final Parameter> languageToMap = mapParam( + "language_to", + true, + m -> builder(m).languageToMap.getValue(), + Collections.emptyMap() + ).neverSerialize(); + + private final Parameter positionIncrementGap = Parameter.intParam( + "position_increment_gap", + true, + m -> builder(m).positionIncrementGap.getValue(), + -1 + ); + + final IndexAnalyzers indexAnalyzers; + final NamedAnalyzer indexAnalyzer; + final NamedAnalyzer searchAnalyzer; + final NamedAnalyzer searchQuoteAnalyzer; + + public Builder(String name, IndexAnalyzers indexAnalyzers) { + super(name); + this.indexAnalyzers = indexAnalyzers; + this.indexAnalyzer = indexAnalyzers.getDefaultIndexAnalyzer(); + this.searchAnalyzer = indexAnalyzers.getDefaultSearchAnalyzer(); + this.searchQuoteAnalyzer = indexAnalyzers.getDefaultSearchQuoteAnalyzer(); + } + + @Override + protected List> getParameters() { + return Arrays.asList(store, positionIncrementGap, + nTrial, alpha, alphaWidth, iterationLimit, probThreshold, convThreshold, + baseFreq, filterPattern, max, binary, map, languages, profile, languageToMap); } public NamedAnalyzer indexAnalyzer() { - return indexAnalyzer; + return indexAnalyzer(); } public NamedAnalyzer searchAnalyzer() { - return searchAnalyzer; + return searchAnalyzer(); } public NamedAnalyzer searchQuoteAnalyzer() { - return searchQuoteAnalyzer; - } - - public Builder positionIncrementGap(int positionIncrementGap) { - this.positionIncrementGap = positionIncrementGap; - return this; - } - - public Builder ntrials(int trials) { - settingsBuilder.put("number_of_trials", trials); - return this; - } - - public Builder alpha(double alpha) { - settingsBuilder.put("alpha", alpha); - return this; - } - - public Builder alphaWidth(double alphaWidth) { - settingsBuilder.put("alpha_width", alphaWidth); - return this; - } - - public Builder iterationLimit(int iterationLimit) { - settingsBuilder.put("iteration_limit", iterationLimit); - return this; - } - - public Builder probThreshold(double probThreshold) { - settingsBuilder.put("prob_threshold", probThreshold); - return this; - } - - public Builder convThreshold(double convThreshold) { - settingsBuilder.put("conv_threshold", convThreshold); - return this; - } - - public Builder baseFreq(int baseFreq) { - settingsBuilder.put("base_freq", baseFreq); - return this; - } - - public Builder pattern(String pattern) { - settingsBuilder.put("pattern", pattern); - return this; - } - - public Builder max(int max) { - settingsBuilder.put("max", max); - return this; - } - - public Builder binary(boolean binary) { - settingsBuilder.put("binary", binary); - return this; - } - - public Builder map(Map map) { - for (Map.Entry entry : map.entrySet()) { - settingsBuilder.put("map." + entry.getKey(), (String) entry.getValue()); - } - return this; - } - - public Builder languages(String[] languages) { - List list = Arrays.asList(languages); - settingsBuilder.putList("languages", list); - return this; - } - - public Builder profile(String profile) { - settingsBuilder.put("profile", profile); - return this; - } - - public Builder languageTo(LanguageTo languageTo) { - this.languageTo = languageTo; - return this; + return searchQuoteAnalyzer(); } @Override - public LangdetectMapper build(BuilderContext context) { - LangdetectService service = new LangdetectService(settingsBuilder.build()); + public LangdetectMapper build(ContentPath context) { return new LangdetectMapper(name, - fieldType, + Defaults.FIELD_TYPE, new LangdetectFieldType(buildFullName(context)), multiFieldsBuilder.build(this, context), - copyTo, - languageTo, - service); + copyTo.build(), + this + ); } } @@ -329,7 +436,6 @@ public static class LangdetectFieldType extends SimpleMappedFieldType { public LangdetectFieldType(String name) { super(name, true, false, true, TextSearchInfo.NONE, Collections.emptyMap()); - setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); } /** Returns the indexed value used to construct search "values". @@ -340,7 +446,7 @@ protected BytesRef indexedValueForSearch(Object value) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + public ValueFetcher valueFetcher(QueryShardContext context, String format) { throw new UnsupportedOperationException(); } @@ -376,110 +482,10 @@ public Query termsQuery(List values, QueryShardContext context) { } - public static class TypeParser implements Mapper.TypeParser { + public static final FieldMapper.TypeParser PARSER + = new FieldMapper.TypeParser((n, c) -> new LangdetectMapper.Builder(n, c.getIndexAnalyzers())); - @Override - public Mapper.Builder parse(String name, Map mapping, ParserContext parserContext) { - Builder builder = new Builder(name); - Iterator> iterator = mapping.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - switch (fieldName) { - case "analyzer": - case "include_in_all": - iterator.remove(); - break; - case "position_increment_gap": - int newPositionIncrementGap = XContentMapValues.nodeIntegerValue(fieldNode, -1); - if (newPositionIncrementGap < 0) { - throw new MapperParsingException("position_increment_gap less than 0 aren't allowed."); - } - builder.positionIncrementGap(newPositionIncrementGap); - if (builder.indexAnalyzer() == null) { - builder.indexAnalyzer(parserContext.getIndexAnalyzers().getDefaultIndexAnalyzer()); - } - if (builder.searchAnalyzer() == null) { - builder.searchAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchAnalyzer()); - } - if (builder.searchQuoteAnalyzer() == null) { - builder.searchQuoteAnalyzer(parserContext.getIndexAnalyzers().getDefaultSearchQuoteAnalyzer()); - } - iterator.remove(); - break; - case "store": - builder.store(XContentMapValues.nodeBooleanValue(fieldNode)); - iterator.remove(); - break; - case "number_of_trials": - builder.ntrials(XContentMapValues.nodeIntegerValue(fieldNode)); - iterator.remove(); - break; - case "alpha": - builder.alpha(XContentMapValues.nodeDoubleValue(fieldNode)); - iterator.remove(); - break; - case "alpha_width": - builder.alphaWidth(XContentMapValues.nodeDoubleValue(fieldNode)); - iterator.remove(); - break; - case "iteration_limit": - builder.iterationLimit(XContentMapValues.nodeIntegerValue(fieldNode)); - iterator.remove(); - break; - case "prob_threshold": - builder.probThreshold(XContentMapValues.nodeDoubleValue(fieldNode)); - iterator.remove(); - break; - case "conv_threshold": - builder.convThreshold(XContentMapValues.nodeDoubleValue(fieldNode)); - iterator.remove(); - break; - case "base_freq": - builder.baseFreq(XContentMapValues.nodeIntegerValue(fieldNode)); - iterator.remove(); - break; - case "pattern": - builder.pattern(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "max": - builder.max(XContentMapValues.nodeIntegerValue(fieldNode)); - iterator.remove(); - break; - case "binary": - builder.binary(XContentMapValues.nodeBooleanValue(fieldNode)); - iterator.remove(); - break; - case "map": - builder.map(XContentMapValues.nodeMapValue(fieldNode, "map")); - iterator.remove(); - break; - case "languages": - builder.languages(XContentMapValues.nodeStringArrayValue(fieldNode)); - iterator.remove(); - break; - case "profile": - builder.profile(XContentMapValues.nodeStringValue(fieldNode, null)); - iterator.remove(); - break; - case "language_to" : - Map map = XContentMapValues.nodeMapValue(fieldNode, null); - LanguageTo.Builder languageToBuilder = LanguageTo.builder(); - languageToBuilder.add(map); - builder.languageTo(languageToBuilder.build()); - iterator.remove(); - break; - default: - break; - } - } - return builder; - } - } - - public static class LanguageTo { + public static class LanguageTo implements ToXContent { private final Map languageToFields; @@ -525,4 +531,30 @@ public Map languageToFields() { } } + public static Parameter> mapParam( + String name, boolean updateable, + Function> initializer, Map defaultValue) + { + return new Parameter<>( + name, + updateable, + () -> defaultValue, + (n, c, o) -> XContentMapValues.nodeMapValue(o, "map"), + initializer + ); + } + + public static Parameter> listParam( + String name, boolean updateable, + Function> initializer, List defaultValue) + { + return new Parameter<>( + name, + updateable, + () -> defaultValue, + (n, c, o) -> Arrays.asList(XContentMapValues.nodeStringArrayValue(o)), + initializer + ); + } + } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java index 9a78481..dc501e1 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapper.java @@ -21,26 +21,28 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextSearchInfo; -import org.elasticsearch.index.mapper.TypeParsers; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; -import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Supplier; /** * Reference field mapper. @@ -48,9 +50,7 @@ public class ReferenceMapper extends FieldMapper { private static final Logger logger = LogManager.getLogger(ReferenceMapper.class.getName()); - public static final String CONTENT_TYPE = "ref"; - public static final FieldType FIELD_TYPE = new FieldType(); static { @@ -59,21 +59,15 @@ public class ReferenceMapper extends FieldMapper { private static final CopyTo COPYTO_EMPTY = new CopyTo.Builder().build(); - private final Client client; - - private String index; - - private String type; - - private List fields; - + private Client client; + private String refIndex; + private String refType; + private List refFields; private final FieldMapper contentMapper; - // override copyTo in FieldMapper private final CopyTo copyTo; public ReferenceMapper(String simpleName, - FieldType fieldType, MappedFieldType defaultFieldType, Client client, String refindex, @@ -81,15 +75,33 @@ public ReferenceMapper(String simpleName, List reffields, FieldMapper contentMapper, MultiFields multiFields, - CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, multiFields, COPYTO_EMPTY); + CopyTo copyTo, + Builder builder) { + super(simpleName, defaultFieldType, Lucene.STANDARD_ANALYZER, multiFields, COPYTO_EMPTY); this.copyTo = copyTo; this.client = client; - this.index = refindex; - this.type = reftype; - this.fields = reffields; + this.refIndex = refindex; + this.refType = reftype; + this.refFields = reffields; this.contentMapper = contentMapper; + this.builder = builder; + } + + private static ReferenceMapper toType(FieldMapper in) { + return (ReferenceMapper) in; + } + + private static ReferenceMapper.Builder builder(FieldMapper in) { + return toType(in).builder; + } + + private final ReferenceMapper.Builder builder; + + /* + public CopyTo copyTo() { + return this.copyTo; } + */ @Override @SuppressWarnings("unchecked") @@ -109,25 +121,25 @@ public void parse(ParseContext originalContext) throws IOException { if (currentFieldName != null) { switch (currentFieldName) { case "ref_index": - index = parser.text(); + refIndex = parser.text(); break; case "ref_type": - type = parser.text(); + refType = parser.text(); break; case "ref_fields": // single field only - fields = new LinkedList<>(); - fields.add(parser.text()); + refFields = new LinkedList<>(); + refFields.add(parser.text()); break; default: break; } } } else if (token == XContentParser.Token.START_ARRAY && "ref_fields".equals(currentFieldName)) { - fields = new LinkedList<>(); + refFields = new LinkedList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { if (parser.text() != null) { - fields.add(parser.text()); + refFields.add(parser.text()); } } } @@ -138,16 +150,16 @@ public void parse(ParseContext originalContext) throws IOException { } context = context.createExternalValueContext(content); contentMapper.parse(context); - if (client != null && index != null && type != null && fields != null) { + if (client != null && refIndex != null && refType != null && refFields != null) { try { GetResponse response = client.prepareGet() - .setIndex(index) - .setType(type) + .setIndex(refIndex) + .setType(refType) .setId(content) .execute() .actionGet(); if (response != null && response.isExists()) { - for (String field : fields) { + for (String field : refFields) { Map source = response.getSource(); List list = XContentMapValues.extractRawValues(field, source); if (list.isEmpty()) { @@ -172,21 +184,22 @@ public void parse(ParseContext originalContext) throws IOException { } } } else { - logger.warn("ref doc does not exist: {}/{}/{}", index, type, content); + logger.warn("ref doc does not exist: {}/{}/{}", refIndex, refType, content); } } catch (Exception e) { - logger.error("error while getting ref doc " + index + "/" + type + "/"+ content + ": " + e.getMessage(), e); + logger.error("error while getting ref doc " + refIndex + "/" + refType + "/" + content + ": " + e.getMessage(), e); } } else { logger.warn("missing prerequisite: client={} index={} type={} fields={}", - client, index, type, fields); + client, refIndex, refType, refFields + ); } - return; } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public FieldMapper.Builder getMergeBuilder() { + //return new Builder(simpleName(), client).init(this); + return null; } @Override @@ -196,17 +209,12 @@ protected void parseCreateField(ParseContext parseContext) { @Override protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { - super.doXContentBody(builder, includeDefaults, params); - if (index != null) { - builder.field("ref_index", index); - } - if (type != null) { - builder.field("ref_type", type); - } - if (fields != null) { - builder.field("ref_fields", fields); - } - if (copyTo != null) { + builder.field("type", contentType()); + Builder mergeBuilder = new Builder(simpleName(), client); + mergeBuilder.init(this); + mergeBuilder.toXContent(builder, includeDefaults); + multiFields.toXContent(builder, params); + if (null != copyTo) { copyTo.toXContent(builder, params); } } @@ -249,6 +257,32 @@ private static void parseCopyFields(ParseContext originalContext, List c } } + public static final class TypeParser implements Mapper.TypeParser { + + private final BiFunction builderFunction; + private Client client; + + /** + * Creates a new TypeParser + * @param builderFunction a function that produces a Builder from a name and parsercontext + */ + public TypeParser(BiFunction builderFunction) { + this.builderFunction = builderFunction; + } + + @Override + public Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + Builder builder = builderFunction.apply(name, parserContext); + this.client = builder.client; + builder.parse(name, parserContext, node); + return builder; + } + + public Client getClient() { + return client; + } + } + public static final class ReferenceFieldType extends SimpleMappedFieldType { public ReferenceFieldType(String name) { @@ -258,7 +292,7 @@ public ReferenceFieldType(String name) { } @Override - public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) { + public ValueFetcher valueFetcher(QueryShardContext context, String format) { throw new UnsupportedOperationException(); } @@ -311,17 +345,31 @@ public Query existsQuery(QueryShardContext context) { } @SuppressWarnings({"rawtypes"}) - public static class Builder extends FieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { private TextFieldMapper.Builder contentBuilder; - private Client client; - private String refIndex; + private final Parameter refIndex = Parameter.stringParam( + "ref_index", + true, + m -> builder(m).refIndex.getValue(), + "" + ); - private String refType; + private final Parameter refType = Parameter.stringParam( + "ref_type", + true, + m -> builder(m).refType.getValue(), + "" + ); - private List refFields; + private final Parameter> refFields = listParam( + "ref_fields", + true, + m -> builder(m).refFields.getValue(), + Collections.emptyList() + ); private static final IndexAnalyzers INDEX_ANALYZERS = new IndexAnalyzers( Collections.singletonMap( @@ -333,90 +381,62 @@ public static class Builder extends FieldMapper.Builder { ); public Builder(String name, Client client) { - super(name, FIELD_TYPE); - this.client = client; - this.refFields = new LinkedList<>(); + super(name); this.contentBuilder = new TextFieldMapper.Builder(name, INDEX_ANALYZERS); + this.client = client; } - public Builder refIndex(String refIndex) { - this.refIndex = refIndex; - return this; - } - - public Builder refType(String refType) { - this.refType = refType; - return this; - } - - @SuppressWarnings("unchecked") - public Builder refFields(Object refFields) { - if (refFields instanceof List) { - this.refFields = (List) refFields; - } else if (refFields != null) { - this.refFields = Collections.singletonList(refFields.toString()); - } - return this; + @Override + public List> getParameters() { + return Arrays.asList(refIndex, refType, refFields); } @Override - public ReferenceMapper build(BuilderContext context) { - FieldMapper contentMapper = (FieldMapper) contentBuilder.build(context); - return new ReferenceMapper(name, - fieldType, - new ReferenceFieldType(buildFullName(context)), - client, - refIndex, - refType, - refFields, - contentMapper, - multiFieldsBuilder.build(this, context), - copyTo); + public ReferenceMapper build(ContentPath contentPath) { + FieldMapper contentMapper = contentBuilder.build(contentPath); + return new ReferenceMapper( + name, + new ReferenceFieldType(buildFullName(contentPath)), + client, + refIndex.getValue(), + refType.getValue(), + refFields.getValue(), + contentMapper, + multiFieldsBuilder.build(this, contentPath), + copyTo.build(), + this + ); } } - public static class TypeParser implements Mapper.TypeParser { - - private Client client; - - public void setClient(Client client) { - this.client = client; - } - - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Mapper.Builder parse(String name, Map node, ParserContext parserContext) - throws MapperParsingException { - ReferenceMapper.Builder builder = new Builder(name, client); - TypeParsers.parseField(builder, name, node, parserContext); - Iterator> iterator = node.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String fieldName = entry.getKey(); - Object fieldNode = entry.getValue(); - switch (fieldName) { - case "analyzer": { - // ignore - iterator.remove(); - break; - } - case "ref_index": - builder.refIndex(fieldNode.toString()); - iterator.remove(); - break; - case "ref_type": - builder.refType(fieldNode.toString()); - iterator.remove(); - break; - case "ref_fields": - builder.refFields(entry.getValue()); - iterator.remove(); - break; - default: - break; - } + public static final ReferenceMapper.TypeParser PARSER = new ReferenceMapper.TypeParser( + (n, c) -> { + Client client = null; + try + { + client = Optional.ofNullable(c) + .map(Mapper.TypeParser.ParserContext::queryShardContextSupplier) + .map(Supplier::get) + .map(QueryShardContext::getClient) + .orElse(null); } - return builder; + catch (Exception ignored) + { + } + return new ReferenceMapper.Builder(n, client); } + ); + + public static Parameter> listParam( + String name, boolean updateable, + Function> initializer, List defaultValue) + { + return new Parameter<>( + name, + updateable, + () -> defaultValue, + (n, c, o) -> Arrays.asList(XContentMapValues.nodeStringArrayValue(o)), + initializer + ); } } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperModule.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperModule.java index 58a1738..1fa910a 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperModule.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperModule.java @@ -1,22 +1,20 @@ package org.xbib.elasticsearch.plugin.bundle.index.mapper.reference; import org.elasticsearch.common.inject.AbstractModule; -import org.xbib.elasticsearch.plugin.bundle.common.reference.ReferenceService; /** * Reference field mapper module. */ public class ReferenceMapperModule extends AbstractModule { - private final ReferenceMapperTypeParser typeParser; + private final ReferenceMapper.TypeParser typeParser; - public ReferenceMapperModule(ReferenceMapperTypeParser typeParser) { + public ReferenceMapperModule(ReferenceMapper.TypeParser typeParser) { this.typeParser = typeParser; } @Override protected void configure() { - bind(ReferenceService.class).asEagerSingleton(); - bind(ReferenceMapperTypeParser.class).toInstance(typeParser); + bind(ReferenceMapper.TypeParser.class).toInstance(typeParser); } } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperTypeParser.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperTypeParser.java deleted file mode 100644 index 76b9509..0000000 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/reference/ReferenceMapperTypeParser.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.xbib.elasticsearch.plugin.bundle.index.mapper.reference; - -/** - * Reference field mapper type parser. - */ -public final class ReferenceMapperTypeParser extends ReferenceMapper.TypeParser { -} diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapper.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapper.java index b7bd6c1..ee5ce35 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapper.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapper.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.index.mapper.ContentPath; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -18,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -36,6 +38,7 @@ public class StandardnumberMapper extends FieldMapper { private final Settings settings; private final StandardnumberService service; + final FieldType fieldType; public StandardnumberMapper(Settings settings, String simpleName, @@ -44,14 +47,15 @@ public StandardnumberMapper(Settings settings, MultiFields multiFields, CopyTo copyTo, StandardnumberService service) { - super(simpleName, fieldType, mappedFieldType, multiFields, copyTo); + super(simpleName, mappedFieldType, multiFields, copyTo); this.settings = settings; this.service = service; + this.fieldType = fieldType; } @Override - protected void mergeOptions(FieldMapper other, List conflicts) { - + public FieldMapper.Builder getMergeBuilder() { + return new StandardnumberMapper.Builder(simpleName(), service).init(this); } @Override @@ -116,42 +120,36 @@ public static final class Defaults { } } - public static class Builder extends FieldMapper.Builder { + public static class Builder extends FieldMapper.Builder { - private Settings.Builder settingsBuilder = Settings.builder(); - private String nullValue; + public Settings.Builder settingsBuilder = Settings.builder(); private final StandardnumberService service; public Builder(String name, StandardnumberService service) { - super(name, FIELD_TYPE); + super(name); this.service = service; - this.builder = this; } public Builder standardNumbers(String[] standardnumbers) { settingsBuilder.putList("standardnumbers", standardnumbers); - return builder; + return this; } - public Builder nullValue(String nullValue) { - this.nullValue = nullValue; - return builder; + @Override + protected List> getParameters() { + return Collections.emptyList(); } @Override - public StandardnumberMapper build(BuilderContext context) { - if (fieldType.indexOptions() != IndexOptions.NONE && !fieldType.tokenized()) { - fieldType.setOmitNorms(true); - fieldType.setIndexOptions(IndexOptions.DOCS); - } + public StandardnumberMapper build(ContentPath context) { TextFieldMapper.TextFieldType ft = new TextFieldMapper.TextFieldType(name); return new StandardnumberMapper(settingsBuilder.build(), name, - fieldType, + FIELD_TYPE, ft, multiFieldsBuilder.build(this, context), - copyTo, + copyTo.build(), service); } } @@ -185,4 +183,8 @@ public Mapper.Builder parse(String name, Map mapping, ParserCont return builder; } } + + public static TypeParser newTypeParser() { + return new TypeParser(); + } } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperModule.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperModule.java index 86419e4..075ac8a 100644 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperModule.java +++ b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperModule.java @@ -8,16 +8,16 @@ */ public class StandardnumberMapperModule extends AbstractModule { - private final StandardnumberMapperTypeParser typeParser; + private final StandardnumberMapper.TypeParser typeParser; - public StandardnumberMapperModule(StandardnumberMapperTypeParser typeParser) { + public StandardnumberMapperModule(StandardnumberMapper.TypeParser typeParser) { this.typeParser = typeParser; } @Override protected void configure() { bind(StandardnumberService.class).asEagerSingleton(); - bind(StandardnumberMapperTypeParser.class).toInstance(typeParser); + bind(StandardnumberMapper.TypeParser.class).toInstance(typeParser); } } diff --git a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperTypeParser.java b/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperTypeParser.java deleted file mode 100644 index 5db329e..0000000 --- a/src/main/java/org/xbib/elasticsearch/plugin/bundle/index/mapper/standardnumber/StandardnumberMapperTypeParser.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.xbib.elasticsearch.plugin.bundle.index.mapper.standardnumber; - -/** - * Standard number field mapper type parser. - */ -public final class StandardnumberMapperTypeParser extends StandardnumberMapper.TypeParser { -} diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 00b041e..2ffaaed 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,4 +1,4 @@ -grant codeBase "${codebase.elasticsearch-plugin-bundle}" { +grant { permission java.io.FilePermission "*", "read"; permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "getClassLoader"; diff --git a/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/langdetect/LangdetectMappingTests.java b/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/langdetect/LangdetectMappingTests.java index e27f97e..e1959c1 100644 --- a/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/langdetect/LangdetectMappingTests.java +++ b/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/langdetect/LangdetectMappingTests.java @@ -68,8 +68,8 @@ public void testBinary() throws Exception { public void testBinary2() throws Exception { String mapping = copyToStringFromClasspath("base64-2-mapping.json"); DocumentMapper docMapper = createIndex("some_index") - .mapperService().documentMapperParser() - .parse("someType", new CompressedXContent(mapping)); + .mapperService() + .parse("someType", new CompressedXContent(mapping), false); //String sampleBinary = copyToStringFromClasspath("base64-2.txt"); String sampleText = copyToStringFromClasspath("base64-2-decoded.txt"); BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() diff --git a/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/standardnumber/StandardnumberMappingTests.java b/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/standardnumber/StandardnumberMappingTests.java index 1ec737d..55e63e8 100644 --- a/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/standardnumber/StandardnumberMappingTests.java +++ b/src/test/java/org/xbib/elasticsearch/plugin/bundle/test/index/mapper/standardnumber/StandardnumberMappingTests.java @@ -32,8 +32,8 @@ protected Collection> getPlugins() { public void testSimpleStandardNumber() throws Exception { String mapping = copyToStringFromClasspath("mapping.json"); DocumentMapper docMapper = createIndex("some_index") - .mapperService().documentMapperParser() - .parse("someType", new CompressedXContent(mapping)); + .mapperService() + .parse("someType", new CompressedXContent(mapping), false); String sampleText = "978-3-551-75213-0"; BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() .field("someField", sampleText).endObject()); @@ -47,8 +47,8 @@ public void testSimpleStandardNumber() throws Exception { public void testNonStandardnumber() throws Exception { String mapping = copyToStringFromClasspath("mapping.json"); DocumentMapper docMapper = createIndex("some_index") - .mapperService().documentMapperParser() - .parse("someType", new CompressedXContent(mapping)); + .mapperService() + .parse("someType", new CompressedXContent(mapping), false); String sampleText = "Hello world"; BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder() .startObject().field("someField", sampleText).endObject()); @@ -59,8 +59,8 @@ public void testNonStandardnumber() throws Exception { String builtMapping = docMapper.mappingSource().string(); logger.warn("testNonStandardnumber: built mapping =" + builtMapping); DocumentMapper docMapper2 = createIndex("some_index2") - .mapperService().documentMapperParser() - .parse("someType", new CompressedXContent(builtMapping)); + .mapperService() + .parse("someType", new CompressedXContent(builtMapping), false); json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() .field("someField", sampleText).endObject()); sourceToParse = new SourceToParse("some_index2", "someType", "1", json, XContentType.JSON); From 5707f0932c0e8779aa796aeafd01f4d67b1d6346 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Swen=20Th=C3=BCmmler?= Date: Thu, 18 Feb 2021 12:09:38 +0100 Subject: [PATCH 3/7] disable checkstyle --- build.gradle | 3 +-- config/checkstyle/checkstyle.xml | 40 ++++++++++++++++---------------- gradle.properties | 1 - 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/build.gradle b/build.gradle index 1baf77f..c365eea 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,6 @@ plugins { id "io.codearte.nexus-staging" version "0.21.1" id "org.xbib.gradle.plugin.jflex" version "1.4.0" id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" - id 'checkstyle' } apply plugin: 'java-library' @@ -221,7 +220,7 @@ asciidoctor { 'source-highlighter': 'coderay' } -checkstyleMain.source = 'src/main/java' +//checkstyleMain.source = 'src/main/java' // we use @Ignore //forbiddenApisTest.enabled = false //tasks.named("forbiddenPatterns").configure { diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml index 52fe33c..9217bfc 100644 --- a/config/checkstyle/checkstyle.xml +++ b/config/checkstyle/checkstyle.xml @@ -38,6 +38,22 @@ page at http://checkstyle.sourceforge.net/config.html --> Only allows a package-info.java, not package.html. --> + + + + + + + + + + @@ -73,12 +89,12 @@ page at http://checkstyle.sourceforge.net/config.html --> - + - - - + + + @@ -184,22 +200,6 @@ page at http://checkstyle.sourceforge.net/config.html --> --> - - - - - - - - - - diff --git a/gradle.properties b/gradle.properties index d15b59b..0b9f76f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -11,7 +11,6 @@ jackson.version = 2.10.4 spatial4j.version = 0.7 jts.version = 1.15.1 jna.version = 5.5.0 -checkstyle.version = 8.13 wagon.version = 3.0.0 log4j.version = 2.13.1 From d900694db29cb9aa00cf72c49be66adeea946719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Fri, 29 Jan 2021 09:41:25 +0100 Subject: [PATCH 4/7] update to ES 7.10.2 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bca17f3..12d38de 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 8500dd1646d9d5dc48a3b9d3a8b833b61a49ef9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Swen=20Th=C3=BCmmler?= Date: Thu, 18 Feb 2021 16:34:16 +0100 Subject: [PATCH 5/7] add changes from upstream 7.10 --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index bca17f3..12d38de 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 5ae168c43677284d0eba6cca66312059b639bc5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Swen=20Th=C3=BCmmler?= Date: Thu, 18 Feb 2021 16:35:05 +0100 Subject: [PATCH 6/7] remove "forbidden" permissions --- src/main/plugin-metadata/plugin-security.policy | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 2ffaaed..ef45830 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,12 +1,10 @@ grant { - permission java.io.FilePermission "*", "read"; permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "getClassLoader"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; }; grant codeBase "${codebase.icu4j}" { - permission java.io.FilePermission "*", "read"; permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.RuntimePermission "getClassLoader"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; From a7cf1fd16395de893043962efaf4e490b149803b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Swen=20Th=C3=BCmmler?= Date: Fri, 19 Feb 2021 13:30:05 +0100 Subject: [PATCH 7/7] update to ES 7.11.1 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 0b9f76f..8381d57 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,8 +1,8 @@ group = org.xbib.elasticsearch name = elasticsearch-plugin-bundle -version = 7.11.0.0 +version = 7.11.1.0 -elasticsearch.version = 7.11.0 +elasticsearch.version = 7.11.1 lucene.version = 8.7.0 icu4j.version = 67.1