From 365bb13d962151ad3285afff717d796ea129abb6 Mon Sep 17 00:00:00 2001 From: pj-cegeka <119848850+pj-cegeka@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:14:04 +0200 Subject: [PATCH] Feat: create version object processor accepts list of member types (#656) --- .../ldio-version-object-creator.md | 8 ++-- .../ldes/ldi/VersionObjectCreator.java | 13 ++++--- .../ldes/ldi/VersionObjectCreatorTest.java | 39 +++++++++++++++---- .../CreateVersionObjectProcessor.java | 4 +- ...ionObjectProcessorPropertyDescriptors.java | 14 +++++-- .../ldes/ldio/LdioVersionObjectCreator.java | 6 ++- .../LdioVersionObjectCreatorAutoConfig.java | 25 ++++++++---- 7 files changed, 77 insertions(+), 32 deletions(-) diff --git a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md index 78837b9ed..83e5810bc 100644 --- a/docs/_ldio/ldio-transformers/ldio-version-object-creator.md +++ b/docs/_ldio/ldio-transformers/ldio-version-object-creator.md @@ -15,9 +15,9 @@ The Version Object Creator will transform a State Object to a Version Object. | Property | Description | Required | Default | Example | Supported values | |:-------------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------------------------------|:------------------------------------------|:-----------------| | _date-observed-property_ | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No | Current Timestamp | \ | String | -| _member-type_ | Defines the RDF type of the version object | No | N/A | https://example.org/Person | String | +| _member-type_ | Defines the RDF type of the object to be transformed to a version object. | Yes | N/A | https://example.org/Person | String | | _delimiter_ | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved) | No | / | / | String | -| _generatedAt-property_ | A statement will be added to the model with the observedAt value and the given property. | No | http://www.w3.org/ns/prov#generatedAtTime | http://www.w3.org/ns/prov#generatedAtTime | String | +| _generatedAt-property_ | A statement will be added to the model with the generatedAt value and the given property. | No | http://www.w3.org/ns/prov#generatedAtTime | http://www.w3.org/ns/prov#generatedAtTime | String | | _versionOf-property_ | A statement will be added to the model with the versionOf value and the given property. | No | http://purl.org/dc/terms/isVersionOf | http://purl.org/dc/terms/isVersionOf | String | ## Example @@ -82,7 +82,9 @@ The YAML configuration of this example would be as follows: transformers: - name: Ldio:VersionObjectCreator config: - member-type: http://example.org/Something + member-type: + - http://example.org/Something + - http://example.org/SomethingElse delimiter: "#" date-observed-property: / generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime diff --git a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java index 9acfe85f5..549e02423 100644 --- a/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java +++ b/ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java @@ -41,7 +41,7 @@ public class VersionObjectCreator implements LdiOneToOneTransformer { /** * Representation of the member type resource that the state object represents */ - private final Resource memberTypeResource; + private final List memberTypeResources; /** * Represents that needs to be used to expand the named subject of the member to a version object member id */ @@ -55,11 +55,11 @@ public class VersionObjectCreator implements LdiOneToOneTransformer { */ private final Property versionOfProperty; - public VersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, Resource memberTypeResource, - String delimiter, - Property generatedAtTimeProperty, Property versionOfProperty) { + public VersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, List memberTypeResources, + String delimiter, + Property generatedAtTimeProperty, Property versionOfProperty) { this.dateObservedPropertyExtractor = dateObservedPropertyExtractor; - this.memberTypeResource = memberTypeResource; + this.memberTypeResources = memberTypeResources; this.delimiter = delimiter; this.generatedAtTimeProperty = generatedAtTimeProperty; this.versionOfProperty = versionOfProperty; @@ -83,7 +83,8 @@ public Model transform(Model linkedDataModel) { } private Optional extractMemberInfo(Model linkedDataModel) { - return linkedDataModel.listStatements(null, SYNTAX_TYPE, memberTypeResource) + return linkedDataModel.listStatements(null, SYNTAX_TYPE, (RDFNode) null) + .filterKeep(statement -> memberTypeResources.contains(statement.getResource())) .nextOptional() .map(Statement::getSubject) .map(Resource::asNode) diff --git a/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java b/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java index b69595439..6d5f89052 100644 --- a/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java +++ b/ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java @@ -131,13 +131,13 @@ void when_dateObservedPropertyIsNested_thenAPropertyPathCanBeProvided() { @ParameterizedTest @ArgumentsSource(JsonLDFileArgumentsProvider.class) - void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime startTestTime, String memberType) + void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime startTestTime, List memberTypes) throws IOException, URISyntaxException { Model model = RDFParserBuilder.create().fromString(getJsonString(fileName)).lang(Lang.JSONLD).toModel(); VersionObjectCreator versionObjectCreator = new VersionObjectCreator(new EmptyPropertyExtractor(), - model.createResource(memberType), + memberTypes.stream().map(model::createResource).toList(), DEFAULT_DELIMITER, null, null); Model versionObject = versionObjectCreator.transform(model); @@ -151,6 +151,25 @@ void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime stmt.getSubject().toString().contains(expectedId + minuteAfterTheTestStarted))); } + @Test + void when_EmptyList_Then_ShouldNotMatch() throws URISyntaxException, IOException { + final LocalDateTime now = LocalDateTime.now(); + Model model = RDFParserBuilder.create().fromString(getJsonString("example-waterqualityobserved.json")).lang(Lang.JSONLD).toModel(); + VersionObjectCreator versionObjectCreator = new VersionObjectCreator(new EmptyPropertyExtractor(), + List.of(), + DEFAULT_DELIMITER, null, null); + + Model versionObject = versionObjectCreator.transform(model); + + final String minuteTheTestStarted = getPartOfLocalDateTime(now); + final String minuteAfterTheTestStarted = getPartOfLocalDateTime(now.plusMinutes(1)); + assertTrue(versionObject.listStatements() + .toList() + .stream() + .noneMatch(stmt -> stmt.getSubject().toString().contains(minuteTheTestStarted) || + stmt.getSubject().toString().contains(minuteAfterTheTestStarted))); + } + @Test void when_memberInfoExtractionFails_warningMessageIsLogged() { ListAppender listAppender = createListAppender(); @@ -229,14 +248,14 @@ private String getJsonString(String resource) throws URISyntaxException, IOExcep } private VersionObjectCreator createVersionObjectCreator(Model inputModel, String dateObservedPath) { - Resource memberType = inputModel.createResource("http://example.org/Something"); + List memberTypes = List.of(inputModel.createResource("http://example.org/Something")); PropertyExtractor dateObservedPropertyExtractor = PropertyPathExtractor.from(dateObservedPath); Property generatedAtTimeProperty = inputModel.createProperty("http://www.w3.org/ns/prov#generatedAtTime"); Property versionOfProperty = inputModel.createProperty("http://purl.org/dc/terms/isVersionOf"); return new VersionObjectCreator( dateObservedPropertyExtractor, - memberType, + memberTypes, DEFAULT_DELIMITER, generatedAtTimeProperty, versionOfProperty @@ -260,15 +279,21 @@ public Stream provideArguments(ExtensionContext context) { Arguments.of("example-waterqualityobserved.json", "urn:ngsi-v2:cot-imec-be:WaterQualityObserved:imec-iow-3orY3reQDK5n3TMpPnLVYR/", now, - "https://uri.etsi.org/ngsi-ld/default-context/WaterQualityObserved"), + List.of("https://uri.etsi.org/ngsi-ld/default-context/WaterQualityObserved")), Arguments.of("example-device.json", "urn:ngsi-v2:cot-imec-be:Device:imec-iow-UR5gEycRuaafxnhvjd9jnU/", now, - "https://uri.etsi.org/ngsi-ld/default-context/Device"), + List.of("https://uri.etsi.org/ngsi-ld/default-context/Device")), + Arguments.of("example-device-model.json", + "urn:ngsi-v2:cot-imec-be:devicemodel:imec-iow-sensor-v0005/", + now, + List.of("https://uri.etsi.org/ngsi-ld/default-context/DeviceModel")), Arguments.of("example-device-model.json", "urn:ngsi-v2:cot-imec-be:devicemodel:imec-iow-sensor-v0005/", now, - "https://uri.etsi.org/ngsi-ld/default-context/DeviceModel")); + List.of("https://www.test.org", + "https://uri.etsi.org/ngsi-ld/default-context/DeviceModel", + "https://www.something.org"))); } } diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java index d85672284..7112fddf8 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java @@ -80,13 +80,13 @@ public void onScheduled(final ProcessContext context) { PropertyExtractor dateObservedPropertyExtractor = dateObservedProperty != null ? PropertyPathExtractor.from(dateObservedProperty) : new EmptyPropertyExtractor(); - Resource memberType = getMemberRdfSyntaxType(context); + List memberTypes = getMemberRdfSyntaxTypes(context); String delimiter = getDelimiter(context); Property versionOfKey = getVersionOfKey(context); Property generatedAtTimeProperty = getGeneratedAtTimeProperty(context); dataDestinationFormat = getDataDestinationFormat(context); - versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter, + versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberTypes, delimiter, generatedAtTimeProperty, versionOfKey); } diff --git a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java index 2fa5b31de..ab1e96111 100644 --- a/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java +++ b/ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java @@ -9,6 +9,9 @@ import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.util.StandardValidators; +import java.util.Arrays; +import java.util.List; + public final class CreateVersionObjectProcessorPropertyDescriptors { private static final String DEFAULT_DATE_OBSERVED_VALUE_RDF_PROPERTY = "https://uri.etsi.org/ngsi-ld/observedAt"; private static final String DEFAULT_DELIMITER = "/"; @@ -21,8 +24,8 @@ private CreateVersionObjectProcessorPropertyDescriptors() { public static final PropertyDescriptor MEMBER_RDF_SYNTAX_TYPE = new PropertyDescriptor.Builder() .name("MEMBER_RDF_SYNTAX_TYPE") - .displayName("IRI to member RDF syntax type") - .description("IRI that declares a http://www.w3.org/1999/02/22-rdf-syntax-ns#type") + .displayName("IRIs to member RDF syntax type") + .description("Comma separated list of IRIs that declare a http://www.w3.org/1999/02/22-rdf-syntax-ns#type of all possible members") .required(true) .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) .build(); @@ -78,8 +81,11 @@ public static String getDateObservedValue(ProcessContext context) { return context.getProperty(DATE_OBSERVED_VALUE_RDF_PROPERTY).getValue(); } - public static Resource getMemberRdfSyntaxType(ProcessContext context) { - return ResourceFactory.createResource(context.getProperty(MEMBER_RDF_SYNTAX_TYPE).getValue()); + public static List getMemberRdfSyntaxTypes(ProcessContext context) { + return Arrays.stream(context.getProperty(MEMBER_RDF_SYNTAX_TYPE).getValue().split(",")) + .map(String::trim) + .map(ResourceFactory::createResource) + .toList(); } public static String getDelimiter(ProcessContext context) { diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java index 72e7de1b0..ffc775d23 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java @@ -7,13 +7,15 @@ import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.Resource; +import java.util.List; + public class LdioVersionObjectCreator extends LdioTransformer { public static final String NAME = "Ldio:VersionObjectCreator"; private final VersionObjectCreator versionObjectCreator; - public LdioVersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, Resource memberType, + public LdioVersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, List memberTypes, String delimiter, Property generatedAtProperty, Property versionOfProperty) { - this.versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter, + this.versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberTypes, delimiter, generatedAtProperty, versionOfProperty); } diff --git a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java index 1b6785784..4059741fd 100644 --- a/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java +++ b/ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java @@ -5,6 +5,7 @@ import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyPathExtractor; import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionObjectCreator; import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator; +import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.ConfigPropertyMissingException; import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer; import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties; import org.apache.jena.rdf.model.Model; @@ -14,7 +15,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import java.util.Optional; +import java.util.List; import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionObjectCreator.NAME; @@ -27,6 +28,11 @@ public LdioTransformerConfigurator ldioConfigurator() { public static class LdioVersionObjectCreatorTransformerConfigurator implements LdioTransformerConfigurator { + public static final String DATE_OBSERVED = "date-observed-property"; + public static final String MEMBER_TYPE = "member-type"; + public static final String DELIMITER = "delimiter"; + public static final String GENERATED_AT = "generatedAt-property"; + public static final String VERSION_OF = "versionOf-property"; public static final String DEFAULT_PROV_GENERATED_AT_TIME = "http://www.w3.org/ns/prov#generatedAtTime"; public static final String DEFAULT_VERSION_OF_KEY = "http://purl.org/dc/terms/isVersionOf"; @@ -34,25 +40,28 @@ public static class LdioVersionObjectCreatorTransformerConfigurator implements L public LdioTransformer configure(ComponentProperties properties) { Model initModel = ModelFactory.createDefaultModel(); - PropertyExtractor dateObservedPropertyExtractor = properties.getOptionalProperty("date-observed-property") + PropertyExtractor dateObservedPropertyExtractor = properties.getOptionalProperty(DATE_OBSERVED) .map(PropertyPathExtractor::from) .map(PropertyExtractor.class::cast) .orElseGet(EmptyPropertyExtractor::new); - Resource memberType = Optional.of(properties.getProperty("member-type")) - .map(initModel::createResource).orElse(null); + List memberTypes = properties.getPropertyList(MEMBER_TYPE).stream() + .map(initModel::createResource).toList(); + if (memberTypes.isEmpty()) { + throw new ConfigPropertyMissingException(properties.getPipelineName(), properties.getComponentName(), MEMBER_TYPE); + } - String delimiter = properties.getOptionalProperty("delimiter").orElse("/"); + String delimiter = properties.getOptionalProperty(DELIMITER).orElse("/"); - Property generatedAtProperty = properties.getOptionalProperty("generatedAt-property") + Property generatedAtProperty = properties.getOptionalProperty(GENERATED_AT) .map(initModel::createProperty) .orElseGet(() -> initModel.createProperty(DEFAULT_PROV_GENERATED_AT_TIME)); - Property versionOfProperty = properties.getOptionalProperty("versionOf-property") + Property versionOfProperty = properties.getOptionalProperty(VERSION_OF) .map(initModel::createProperty) .orElseGet(() -> initModel.createProperty(DEFAULT_VERSION_OF_KEY)); - return new LdioVersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter, + return new LdioVersionObjectCreator(dateObservedPropertyExtractor, memberTypes, delimiter, generatedAtProperty, versionOfProperty); }