Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parse the contents of dynamic objects for [subobjects:false] #117762

Merged
merged 15 commits into from
Dec 3, 2024
6 changes: 6 additions & 0 deletions docs/changelog/117762.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 117762
summary: "Parse the contents of dynamic objects for [subobjects:false]"
area: Mapping
type: bug
issues:
- 117544
Original file line number Diff line number Diff line change
Expand Up @@ -1177,3 +1177,121 @@ fetch geo_point:
- is_false: hits.hits.0.fields.message
- match: { hits.hits.0._source.message.foo: 10 }
- match: { hits.hits.0._source.message.foo\.bar: 20 }

---
root with subobjects false and dynamic false:
- requires:
cluster_features: mapper.fix_parsing_subobjects_false_dynamic_false
reason: bug fix

- do:
indices.create:
index: test
body:
mappings:
subobjects: false
dynamic: false
properties:
id:
type: integer
my.keyword.field:
type: keyword

- do:
bulk:
index: test
refresh: true
body:
- '{ "index": { } }'
- '{ "id": 1, "my": { "keyword.field": "abc" } }'
- match: { errors: false }

# indexing a dynamically-mapped field still fails (silently)
- do:
bulk:
index: test
refresh: true
body:
- '{ "index": { } }'
- '{ "id": 2, "my": { "random.field": "abc" } }'
- match: { errors: false }

- do:
search:
index: test
body:
sort: id
fields: [ "*" ]

- match: { hits.hits.0.fields: { my.keyword.field: [ abc ], id: [ 1 ] } }
- match: { hits.hits.1.fields: { id: [ 2 ] } }

- do:
search:
index: test
body:
query:
match:
my.keyword.field: abc

- match: { hits.total.value: 1 }

---
object with subobjects false and dynamic false:
- requires:
cluster_features: mapper.fix_parsing_subobjects_false_dynamic_false
reason: bug fix

- do:
indices.create:
index: test
body:
mappings:
properties:
my:
subobjects: false
dynamic: false
properties:
id:
type: integer
nested.keyword.field:
type: keyword

- do:
bulk:
index: test
refresh: true
body:
- '{ "index": { } }'
- '{ "id": 1, "my": { "nested": { "keyword.field": "abc" } } }'
- match: { errors: false }

# indexing a dynamically-mapped field still fails (silently)
- do:
bulk:
index: test
refresh: true
body:
- '{ "index": { } }'
- '{ "id": 2, "my": { "nested": { "random.field": "abc" } } }'
- match: { errors: false }

- do:
search:
index: test
body:
sort: id
fields: [ "*" ]

- match: { hits.hits.0.fields: { my.nested.keyword.field: [ abc ], id: [ 1 ] } }
- match: { hits.hits.1.fields: { id: [ 2 ] } }

- do:
search:
index: test
body:
query:
match:
my.nested.keyword.field: abc

- match: { hits.total.value: 1 }
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.features.NodeFeature;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.fielddata.FieldDataContext;
Expand Down Expand Up @@ -53,6 +54,9 @@
public final class DocumentParser {

public static final IndexVersion DYNAMICALLY_MAP_DENSE_VECTORS_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION;
static final NodeFeature FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE = new NodeFeature(
"mapper.fix_parsing_subobjects_false_dynamic_false"
);

private final XContentParserConfiguration parserConfiguration;
private final MappingParserContext mappingParserContext;
Expand Down Expand Up @@ -531,7 +535,8 @@ private static void doParseObject(DocumentParserContext context, String currentF

private static void parseObjectDynamic(DocumentParserContext context, String currentFieldName) throws IOException {
ensureNotStrict(context, currentFieldName);
if (context.dynamic() == ObjectMapper.Dynamic.FALSE) {
// For [subobjects:false], intermediate objects get flattened so we can't skip parsing children.
if (context.dynamic() == ObjectMapper.Dynamic.FALSE && context.parent().subobjects() != ObjectMapper.Subobjects.DISABLED) {
failIfMatchesRoutingPath(context, currentFieldName);
if (context.canAddIgnoredField()) {
context.addIgnoredField(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public Set<NodeFeature> getTestFeatures() {
IgnoredSourceFieldMapper.IGNORED_SOURCE_AS_TOP_LEVEL_METADATA_ARRAY_FIELD,
IgnoredSourceFieldMapper.ALWAYS_STORE_OBJECT_ARRAYS_IN_NESTED_OBJECTS,
MapperService.LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT,
DocumentParser.FIX_PARSING_SUBOBJECTS_FALSE_DYNAMIC_FALSE,
CONSTANT_KEYWORD_SYNTHETIC_SOURCE_WRITE_FIX,
META_FETCH_FIELDS_ERROR_CODE_CHANGED
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2053,6 +2053,38 @@ public void testSubobjectsFalseWithInnerDottedObject() throws Exception {
assertNotNull(doc.rootDoc().getField("metrics.service.test.with.dots.max"));
}

public void testSubobjectsFalseWithInnerDottedObjectDynamicFalse() throws Exception {
DocumentMapper mapper = createDocumentMapper(mapping(b -> {
b.startObject("metrics").field("type", "object").field("subobjects", false).field("dynamic", randomFrom("false", "runtime"));
b.startObject("properties").startObject("service.test.with.dots").field("type", "keyword").endObject().endObject();
b.endObject();
}));

ParsedDocument doc = mapper.parse(source("""
{ "metrics": { "service": { "test.with.dots": "foo" } } }"""));
assertNotNull(doc.rootDoc().getField("metrics.service.test.with.dots"));

doc = mapper.parse(source("""
{ "metrics": { "service.test": { "with.dots": "foo" } } }"""));
assertNotNull(doc.rootDoc().getField("metrics.service.test.with.dots"));

doc = mapper.parse(source("""
{ "metrics": { "service": { "test": { "with.dots": "foo" } } } }"""));
assertNotNull(doc.rootDoc().getField("metrics.service.test.with.dots"));

doc = mapper.parse(source("""
{ "metrics": { "service": { "test.other.dots": "foo" } } }"""));
assertNull(doc.rootDoc().getField("metrics.service.test.other.dots"));

doc = mapper.parse(source("""
{ "metrics": { "service.test": { "other.dots": "foo" } } }"""));
assertNull(doc.rootDoc().getField("metrics.service.test.other.dots"));

doc = mapper.parse(source("""
{ "metrics": { "service": { "test": { "other.dots": "foo" } } } }"""));
assertNull(doc.rootDoc().getField("metrics.service.test.other.dots"));
}

public void testSubobjectsFalseRoot() throws Exception {
DocumentMapper mapper = createDocumentMapper(mappingNoSubobjects(xContentBuilder -> {}));
ParsedDocument doc = mapper.parse(source("""
Expand All @@ -2074,6 +2106,37 @@ public void testSubobjectsFalseRoot() throws Exception {
assertNotNull(doc.rootDoc().getField("metrics.service.test.with.dots"));
}

public void testSubobjectsFalseRootWithInnerDottedObjectDynamicFalse() throws Exception {
DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
b.field("subobjects", false).field("dynamic", randomFrom("false", "runtime"));
b.startObject("properties").startObject("service.test.with.dots").field("type", "keyword").endObject().endObject();
}));

ParsedDocument doc = mapper.parse(source("""
{ "service": { "test.with.dots": "foo" } }"""));
assertNotNull(doc.rootDoc().getField("service.test.with.dots"));

doc = mapper.parse(source("""
{ "service.test": { "with.dots": "foo" } }"""));
assertNotNull(doc.rootDoc().getField("service.test.with.dots"));

doc = mapper.parse(source("""
{ "service": { "test": { "with.dots": "foo" } } }"""));
assertNotNull(doc.rootDoc().getField("service.test.with.dots"));

doc = mapper.parse(source("""
{ "service": { "test.other.dots": "foo" } }"""));
assertNull(doc.rootDoc().getField("service.test.other.dots"));

doc = mapper.parse(source("""
{ "service.test": { "other.dots": "foo" } }"""));
assertNull(doc.rootDoc().getField("service.test.other.dots"));

doc = mapper.parse(source("""
{ "service": { "test": { "other.dots": "foo" } } }"""));
assertNull(doc.rootDoc().getField("service.test.other.dots"));
}

public void testSubobjectsFalseStructuredPath() throws Exception {
DocumentMapper mapper = createDocumentMapper(
mapping(b -> b.startObject("metrics.service").field("type", "object").field("subobjects", false).endObject())
Expand Down