Skip to content

Commit

Permalink
implement option to get and apply index settings (#21054)
Browse files Browse the repository at this point in the history
* implement option to get and apply index settings

* fix license header

* fix review findings
  • Loading branch information
AntonEbel authored Nov 28, 2024
1 parent 4ba93ff commit 28d53da
Show file tree
Hide file tree
Showing 8 changed files with 240 additions and 33 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,9 @@
import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.CreateIndexRequest;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.DeleteAliasRequest;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.GetMappingsRequest;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.GetMappingsResponse;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.client.indices.PutMappingRequest;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.cluster.metadata.AliasMetadata;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.common.settings.Settings;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.index.query.QueryBuilders;
import org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse;
Expand Down Expand Up @@ -169,10 +169,7 @@ public void create(String index, IndexSettings indexSettings, @Nullable Map<Stri
private CreateIndexRequest createIndexRequest(String index,
IndexSettings indexSettings,
@Nullable Map<String, Object> mapping) {
final Map<String, Object> settings = new HashMap<>();
settings.put("number_of_shards", indexSettings.shards());
settings.put("number_of_replicas", indexSettings.replicas());
CreateIndexRequest request = new CreateIndexRequest(index).settings(settings);
CreateIndexRequest request = new CreateIndexRequest(index).settings(indexSettings.map());
if (mapping != null) {
request = request.mapping(mapping);
}
Expand Down Expand Up @@ -202,10 +199,24 @@ public Map<String, Object> getIndexMapping(@Nonnull String index) {
.indices(index)
.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);

final GetMappingsResponse result = client.execute((c, requestOptions) -> c.indices().getMapping(request, requestOptions),
return client.execute((c, requestOptions) -> c.indices().getMapping(request, requestOptions).mappings().get(index).sourceAsMap(),
"Couldn't read mapping of index " + index);
}

@Override
public Map<String, Object> getFlattenIndexSettings(@Nonnull String index) {

return result.mappings().get(index).sourceAsMap();
final GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices(index)
.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);

return client.execute((c, requestOptions) -> {
final GetSettingsResponse settingsResponse = c.indices().getSettings(getSettingsRequest, requestOptions);
Settings settings = settingsResponse.getIndexToSettings().get(index);
ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
settings.keySet().forEach(k -> Optional.ofNullable(settings.get(k)).ifPresent(v -> builder.put(k, v)));
return builder.build();
}, "Couldn't read settings of index " + index);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.CreateIndexRequest;
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.DeleteAliasRequest;
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.GetMappingsRequest;
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.GetMappingsResponse;
import org.graylog.shaded.opensearch2.org.opensearch.client.indices.PutMappingRequest;
import org.graylog.shaded.opensearch2.org.opensearch.cluster.metadata.AliasMetadata;
import org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings;
Expand Down Expand Up @@ -168,12 +167,9 @@ public void create(String index, IndexSettings indexSettings, @Nullable Map<Stri
}

private CreateIndexRequest createIndexRequest(String index,
IndexSettings indexSettings,
@Nullable Map<String, Object> mapping) {
final Map<String, Object> settings = new HashMap<>();
settings.put("number_of_shards", indexSettings.shards());
settings.put("number_of_replicas", indexSettings.replicas());
CreateIndexRequest request = new CreateIndexRequest(index).settings(settings);
IndexSettings indexSettings,
@Nullable Map<String, Object> mapping) {
CreateIndexRequest request = new CreateIndexRequest(index).settings(indexSettings.map());
if (mapping != null) {
request = request.mapping(mapping);
}
Expand Down Expand Up @@ -203,12 +199,27 @@ public Map<String, Object> getIndexMapping(@Nonnull String index) {
.indices(index)
.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);

final GetMappingsResponse result = client.execute((c, requestOptions) -> c.indices().getMapping(request, requestOptions),
return client.execute((c, requestOptions) -> c.indices().getMapping(request, requestOptions).mappings().get(index).sourceAsMap(),
"Couldn't read mapping of index " + index);
}

@Override
public Map<String, Object> getFlattenIndexSettings(@Nonnull String index) {

final GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
.indices(index)
.indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);

return result.mappings().get(index).sourceAsMap();
return client.execute((c, requestOptions) -> {
final GetSettingsResponse settingsResponse = c.indices().getSettings(getSettingsRequest, requestOptions);
Settings settings = settingsResponse.getIndexToSettings().get(index);
ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
settings.keySet().forEach(k -> Optional.ofNullable(settings.get(k)).ifPresent(v -> builder.put(k, v)));
return builder.build();
}, "Couldn't read settings of index " + index);
}


@Override
public void updateIndexMetaData(@Nonnull String index, @Nonnull Map<String, Object> metadata, boolean mergeExisting) {
Map<String, Object> metaUpdate = new HashMap<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,18 @@ public Template messageTemplate(final String indexPattern,
final CustomFieldMappings customFieldMappings) {
var settings = new Template.Settings(Map.of(
"index", Map.of(
"analysis", Map.of("analyzer", analyzerKeyword())
"analysis", analysisSettings()
)
));
var mappings = mapping(analyzer, customFieldMappings);

return createTemplate(indexPattern, order, settings, mappings);
}

public Map<String, Object> analysisSettings() {
return Map.of("analyzer", analyzerKeyword());
}

Template createTemplate(String indexPattern, Long order, Template.Settings settings, Template.Mappings mappings) {
return Template.create(indexPattern, mappings, order, settings);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@
*/
package org.graylog2.indexer;

import jakarta.annotation.Nullable;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetMappingTemplate;
import org.graylog2.indexer.indices.IndexSettings;
import org.graylog2.indexer.indices.Template;

import java.util.Map;

/**
* Implementing classes provide an index mapping template representation that can be stored in Elasticsearch.
*/
Expand All @@ -41,4 +46,22 @@ public interface IndexMappingTemplate {
default Template toTemplate(IndexSetMappingTemplate indexSetConfig) {
return toTemplate(indexSetConfig, -1L);
}

default IndexSettings indexSettings(IndexSetConfig indexSetConfig, @Nullable Map<String, Object> settings) {
return createIndexSettings(indexSetConfig);
}

@Nullable
default Map<String, Object> indexMappings(IndexSetConfig indexSetConfig, @Nullable Map<String, Object> mappings) {
return null;
}

static IndexSettings createIndexSettings(IndexSetConfig indexSetConfig) {
return IndexSettings.create(
indexSetConfig.shards(),
indexSetConfig.replicas(),
null
);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,29 @@
*/
package org.graylog2.indexer.indices;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.google.auto.value.AutoValue;
import com.google.common.collect.ImmutableMap;
import jakarta.annotation.Nullable;

@AutoValue
@JsonAutoDetect
public abstract class IndexSettings {
public abstract int shards();
public abstract int replicas();
import java.util.Map;

public static IndexSettings create(int shards, int replicas) {
return new AutoValue_IndexSettings(shards, replicas);
public class IndexSettings {
private final Map<String, Object> settings;

public IndexSettings(Map<String, Object> settings) {
this.settings = settings;
}

public static IndexSettings create(int shards, int replicas, @Nullable Map<String, Object> analysis) {
ImmutableMap.Builder<String, Object> fields = ImmutableMap.<String, Object>builder()
.put("number_of_shards", shards)
.put("number_of_replicas", replicas);
if (analysis != null) {
fields.put("analysis", analysis);
}
return new IndexSettings(fields.build());
}

public Map<String, Object> map() {
return settings;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog2.indexer.indices;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;

public class IndexSettingsHelper {

/**
* This method is copied from {@link org.graylog.shaded.opensearch2.org.opensearch.common.settings.Settings} and adapted.
*/
public static Map<String, Object> getAsStructuredMap(Map<String, Object> settings) {
Map<String, Object> map = new HashMap<>(2);
for (Map.Entry<String, Object> entry : settings.entrySet()) {
processSetting(map, "", entry.getKey(), entry.getValue());
}
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> valMap = (Map<String, Object>) entry.getValue();
entry.setValue(convertMapsToArrays(valMap));
}
}

return map;
}

private static Object convertMapsToArrays(Map<String, Object> map) {
if (map.isEmpty()) {
return map;
}
boolean isArray = true;
int maxIndex = -1;
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (isArray) {
try {
int index = Integer.parseInt(entry.getKey());
if (index >= 0) {
maxIndex = Math.max(maxIndex, index);
} else {
isArray = false;
}
} catch (NumberFormatException ex) {
isArray = false;
}
}
if (entry.getValue() instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> valMap = (Map<String, Object>) entry.getValue();
entry.setValue(convertMapsToArrays(valMap));
}
}
if (isArray && (maxIndex + 1) == map.size()) {
ArrayList<Object> newValue = new ArrayList<>(maxIndex + 1);
for (int i = 0; i <= maxIndex; i++) {
Object obj = map.get(Integer.toString(i));
if (obj == null) {
// Something went wrong. Different format?
// Bailout!
return map;
}
newValue.add(obj);
}
return newValue;
}
return map;
}


private static void processSetting(Map<String, Object> map, String prefix, String setting, Object value) {
int prefixLength = setting.indexOf('.');
if (prefixLength == -1) {
@SuppressWarnings("unchecked")
Map<String, Object> innerMap = (Map<String, Object>) map.get(prefix + setting);
if (innerMap != null) {
// It supposed to be a value, but we already have a map stored, need to convert this map to "." notation
for (Map.Entry<String, Object> entry : innerMap.entrySet()) {
map.put(prefix + setting + "." + entry.getKey(), entry.getValue());
}
}
map.put(prefix + setting, value);
} else {
String key = setting.substring(0, prefixLength);
String rest = setting.substring(prefixLength + 1);
Object existingValue = map.get(prefix + key);
if (existingValue == null) {
Map<String, Object> newMap = new HashMap<>(2);
processSetting(newMap, "", rest, value);
map.put(prefix + key, newMap);
} else {
if (existingValue instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, Object> innerMap = (Map<String, Object>) existingValue;
processSetting(innerMap, "", rest, value);
map.put(prefix + key, innerMap);
} else {
// It supposed to be a map, but we already have a value stored, which is not a map
// fall back to "." notation
processSetting(map, prefix + key + ".", rest, value);
}
}
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.eventbus.EventBus;
import jakarta.annotation.Nullable;
import jakarta.inject.Inject;
import jakarta.inject.Singleton;
import jakarta.validation.constraints.NotNull;
Expand All @@ -31,6 +32,7 @@
import org.graylog2.indexer.ElasticsearchException;
import org.graylog2.indexer.IgnoreIndexTemplate;
import org.graylog2.indexer.IndexMappingFactory;
import org.graylog2.indexer.IndexMappingTemplate;
import org.graylog2.indexer.IndexNotFoundException;
import org.graylog2.indexer.IndexSet;
import org.graylog2.indexer.IndexTemplateNotFoundException;
Expand Down Expand Up @@ -225,18 +227,26 @@ public void deleteIndexTemplate(IndexSet indexSet) {
}

public boolean create(String indexName, IndexSet indexSet) {
return create(indexName, indexSet, null );
return create(indexName, indexSet, null, null );
}

public boolean create(String indexName, IndexSet indexSet, Map<String, Object> indexMapping) {
IndexSettings indexSettings = IndexSettings.create(
indexSet.getConfig().shards(),
indexSet.getConfig().replicas()
);
public boolean create(String indexName,
IndexSet indexSet,
@Nullable Map<String, Object> indexMapping,
@Nullable Map<String, Object> indexSettings) {
try {
// Make sure our index template exists before creating an index!
ensureIndexTemplate(indexSet);
indicesAdapter.create(indexName, indexSettings, indexMapping);
Optional<IndexMappingTemplate> indexMappingTemplate = indexMapping(indexSet);
IndexSettings settings = indexMappingTemplate
.map(t -> t.indexSettings(indexSet.getConfig(), indexSettings))
.orElse(IndexMappingTemplate.createIndexSettings(indexSet.getConfig()));

Map<String, Object> mappings = indexMappingTemplate
.map(t -> t.indexMappings(indexSet.getConfig(), indexMapping))
.orElse(null);

indicesAdapter.create(indexName, settings, mappings);
} catch (Exception e) {
LOG.warn("Couldn't create index {}. Error: {}", indexName, e.getMessage(), e);
auditEventSender.failure(AuditActor.system(nodeId), ES_INDEX_CREATE, ImmutableMap.of("indexName", indexName));
Expand All @@ -246,6 +256,14 @@ public boolean create(String indexName, IndexSet indexSet, Map<String, Object> i
return true;
}

private Optional<IndexMappingTemplate> indexMapping(IndexSet indexSet) {
try {
return Optional.of(indexMappingFactory.createIndexMapping(indexSet.getConfig()));
}catch (IgnoreIndexTemplate e){
return Optional.empty();
}
}

public IndexSetMappingTemplate getTemplateIndexSetConfig(
final IndexSet indexSet,
final IndexSetConfig indexSetConfig,
Expand Down Expand Up @@ -428,4 +446,8 @@ public void refresh(String... indices) {
public Map<String, Object> indexMapping(String index) {
return indicesAdapter.getIndexMapping(index);
}

public Map<String, Object> indexSettings(String index) {
return IndexSettingsHelper.getAsStructuredMap(indicesAdapter.getFlattenIndexSettings(index));
}
}
Loading

0 comments on commit 28d53da

Please sign in to comment.