Feat: create version object processor accepts list of member types (#656

)

docs/_ldio/ldio-transformers/ldio-version-object-creator.md

@@ -15,9 +15,9 @@ The Version Object Creator will transform a State Object to a Version Object.
 | Property                 | Description                                                                                                                    | Required | Default                                   | Example                                   | Supported values |
 |:-------------------------|:-------------------------------------------------------------------------------------------------------------------------------|:---------|:------------------------------------------|:------------------------------------------|:-----------------|
 | _date-observed-property_ | Property path (IRI format '<>') that points to a literal which should be used as timestampPath. Defaults to current timestamp. | No       | Current Timestamp                         | \<https://example.org/ObservedAt\>        | String           |
-| _member-type_            | Defines the RDF type of the version object                                                                                     | No       | N/A                                       | https://example.org/Person                | String           |
+| _member-type_            | Defines the RDF type of the object to be transformed to a version object.                                                      | Yes      | N/A                                       | https://example.org/Person                | String           |
 | _delimiter_              | Defines how the version object id will be constructed. (versionOf + delimiter + dateObserved)                                  | No       | /                                         | /                                         | String           |
-| _generatedAt-property_   | A statement will be added to the model with the observedAt value and the given property.                                       | No       | http://www.w3.org/ns/prov#generatedAtTime | http://www.w3.org/ns/prov#generatedAtTime | String           |
+| _generatedAt-property_   | A statement will be added to the model with the generatedAt value and the given property.                                      | No       | http://www.w3.org/ns/prov#generatedAtTime | http://www.w3.org/ns/prov#generatedAtTime | String           |
 | _versionOf-property_     | A statement will be added to the model with the versionOf value and the given property.                                        | No       | http://purl.org/dc/terms/isVersionOf      | http://purl.org/dc/terms/isVersionOf      | String           |
 
 ## Example
@@ -82,7 +82,9 @@ The YAML configuration of this example would be as follows:
 transformers:
   - name: Ldio:VersionObjectCreator
     config:
-      member-type: http://example.org/Something
+      member-type: 
+        - http://example.org/Something
+        - http://example.org/SomethingElse
       delimiter: "#"
       date-observed-property: <http://example.org/created>/<http://www.w3.org/2006/time#inXSDDateTimeStamp>
       generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime

ldi-core/version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreator.java

@@ -41,7 +41,7 @@ public class VersionObjectCreator implements LdiOneToOneTransformer {
 	/**
 	 * Representation of the member type resource that the state object represents
 	 */
-	private final Resource memberTypeResource;
+	private final List<Resource> memberTypeResources;
 	/**
 	 * Represents that needs to be used to expand the named subject of the member to a version object member id
 	 */
@@ -55,11 +55,11 @@ public class VersionObjectCreator implements LdiOneToOneTransformer {
 	 */
 	private final Property versionOfProperty;
 
-	public VersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, Resource memberTypeResource,
-								String delimiter,
-								Property generatedAtTimeProperty, Property versionOfProperty) {
+	public VersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, List<Resource> memberTypeResources,
+	                            String delimiter,
+	                            Property generatedAtTimeProperty, Property versionOfProperty) {
 		this.dateObservedPropertyExtractor = dateObservedPropertyExtractor;
-		this.memberTypeResource = memberTypeResource;
+		this.memberTypeResources = memberTypeResources;
 		this.delimiter = delimiter;
 		this.generatedAtTimeProperty = generatedAtTimeProperty;
 		this.versionOfProperty = versionOfProperty;
@@ -83,7 +83,8 @@ public Model transform(Model linkedDataModel) {
 	}
 
 	private Optional<String> extractMemberInfo(Model linkedDataModel) {
-		return linkedDataModel.listStatements(null, SYNTAX_TYPE, memberTypeResource)
+		return linkedDataModel.listStatements(null, SYNTAX_TYPE, (RDFNode) null)
+				.filterKeep(statement -> memberTypeResources.contains(statement.getResource()))
 				.nextOptional()
 				.map(Statement::getSubject)
 				.map(Resource::asNode)

ldi-core/version-object-creator/src/test/java/be/vlaanderen/informatievlaanderen/ldes/ldi/VersionObjectCreatorTest.java

@@ -131,13 +131,13 @@ void when_dateObservedPropertyIsNested_thenAPropertyPathCanBeProvided() {
 
 	@ParameterizedTest
 	@ArgumentsSource(JsonLDFileArgumentsProvider.class)
-	void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime startTestTime, String memberType)
+	void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime startTestTime, List<String> memberTypes)
 			throws IOException, URISyntaxException {
 
 		Model model = RDFParserBuilder.create().fromString(getJsonString(fileName)).lang(Lang.JSONLD).toModel();
 
 		VersionObjectCreator versionObjectCreator = new VersionObjectCreator(new EmptyPropertyExtractor(),
-				model.createResource(memberType),
+				memberTypes.stream().map(model::createResource).toList(),
 				DEFAULT_DELIMITER, null, null);
 
 		Model versionObject = versionObjectCreator.transform(model);
@@ -151,6 +151,25 @@ void shouldMatchCountOfObjects(String fileName, String expectedId, LocalDateTime
 						stmt.getSubject().toString().contains(expectedId + minuteAfterTheTestStarted)));
 	}
 
+	@Test
+	void when_EmptyList_Then_ShouldNotMatch() throws URISyntaxException, IOException {
+		final LocalDateTime now = LocalDateTime.now();
+		Model model = RDFParserBuilder.create().fromString(getJsonString("example-waterqualityobserved.json")).lang(Lang.JSONLD).toModel();
+		VersionObjectCreator versionObjectCreator = new VersionObjectCreator(new EmptyPropertyExtractor(),
+				List.of(),
+				DEFAULT_DELIMITER, null, null);
+
+		Model versionObject = versionObjectCreator.transform(model);
+
+		final String minuteTheTestStarted = getPartOfLocalDateTime(now);
+		final String minuteAfterTheTestStarted = getPartOfLocalDateTime(now.plusMinutes(1));
+		assertTrue(versionObject.listStatements()
+				.toList()
+				.stream()
+				.noneMatch(stmt -> stmt.getSubject().toString().contains(minuteTheTestStarted) ||
+						stmt.getSubject().toString().contains(minuteAfterTheTestStarted)));
+	}
+
 	@Test
 	void when_memberInfoExtractionFails_warningMessageIsLogged() {
 		ListAppender<ILoggingEvent> listAppender = createListAppender();
@@ -229,14 +248,14 @@ private String getJsonString(String resource) throws URISyntaxException, IOExcep
 	}
 
 	private VersionObjectCreator createVersionObjectCreator(Model inputModel, String dateObservedPath) {
-		Resource memberType = inputModel.createResource("http://example.org/Something");
+		List<Resource> memberTypes = List.of(inputModel.createResource("http://example.org/Something"));
 		PropertyExtractor dateObservedPropertyExtractor = PropertyPathExtractor.from(dateObservedPath);
 		Property generatedAtTimeProperty = inputModel.createProperty("http://www.w3.org/ns/prov#generatedAtTime");
 		Property versionOfProperty = inputModel.createProperty("http://purl.org/dc/terms/isVersionOf");
 
 		return new VersionObjectCreator(
 				dateObservedPropertyExtractor,
-				memberType,
+				memberTypes,
 				DEFAULT_DELIMITER,
 				generatedAtTimeProperty,
 				versionOfProperty
@@ -260,15 +279,21 @@ public Stream<? extends Arguments> provideArguments(ExtensionContext context) {
 					Arguments.of("example-waterqualityobserved.json",
 							"urn:ngsi-v2:cot-imec-be:WaterQualityObserved:imec-iow-3orY3reQDK5n3TMpPnLVYR/",
 							now,
-							"https://uri.etsi.org/ngsi-ld/default-context/WaterQualityObserved"),
+							List.of("https://uri.etsi.org/ngsi-ld/default-context/WaterQualityObserved")),
 					Arguments.of("example-device.json",
 							"urn:ngsi-v2:cot-imec-be:Device:imec-iow-UR5gEycRuaafxnhvjd9jnU/",
 							now,
-							"https://uri.etsi.org/ngsi-ld/default-context/Device"),
+							List.of("https://uri.etsi.org/ngsi-ld/default-context/Device")),
+					Arguments.of("example-device-model.json",
+							"urn:ngsi-v2:cot-imec-be:devicemodel:imec-iow-sensor-v0005/",
+							now,
+							List.of("https://uri.etsi.org/ngsi-ld/default-context/DeviceModel")),
 					Arguments.of("example-device-model.json",
 							"urn:ngsi-v2:cot-imec-be:devicemodel:imec-iow-sensor-v0005/",
 							now,
-							"https://uri.etsi.org/ngsi-ld/default-context/DeviceModel"));
+							List.of("https://www.test.org",
+									"https://uri.etsi.org/ngsi-ld/default-context/DeviceModel",
+									"https://www.something.org")));
 		}
 	}
 

ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/CreateVersionObjectProcessor.java

@@ -80,13 +80,13 @@ public void onScheduled(final ProcessContext context) {
 		PropertyExtractor dateObservedPropertyExtractor = dateObservedProperty != null
 				? PropertyPathExtractor.from(dateObservedProperty)
 				: new EmptyPropertyExtractor();
-		Resource memberType = getMemberRdfSyntaxType(context);
+		List<Resource> memberTypes = getMemberRdfSyntaxTypes(context);
 		String delimiter = getDelimiter(context);
 		Property versionOfKey = getVersionOfKey(context);
 		Property generatedAtTimeProperty = getGeneratedAtTimeProperty(context);
 		dataDestinationFormat = getDataDestinationFormat(context);
 
-		versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter,
+		versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberTypes, delimiter,
 				generatedAtTimeProperty,
 				versionOfKey);
 	}

ldi-nifi/ldi-nifi-processors/create-version-object-processor/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/processors/config/CreateVersionObjectProcessorPropertyDescriptors.java

@@ -9,6 +9,9 @@
 import org.apache.nifi.processor.ProcessContext;
 import org.apache.nifi.processor.util.StandardValidators;
 
+import java.util.Arrays;
+import java.util.List;
+
 public final class CreateVersionObjectProcessorPropertyDescriptors {
 	private static final String DEFAULT_DATE_OBSERVED_VALUE_RDF_PROPERTY = "https://uri.etsi.org/ngsi-ld/observedAt";
 	private static final String DEFAULT_DELIMITER = "/";
@@ -21,8 +24,8 @@ private CreateVersionObjectProcessorPropertyDescriptors() {
 
 	public static final PropertyDescriptor MEMBER_RDF_SYNTAX_TYPE = new PropertyDescriptor.Builder()
 			.name("MEMBER_RDF_SYNTAX_TYPE")
-			.displayName("IRI to member RDF syntax type")
-			.description("IRI that declares a http://www.w3.org/1999/02/22-rdf-syntax-ns#type")
+			.displayName("IRIs to member RDF syntax type")
+			.description("Comma separated list of IRIs that declare a http://www.w3.org/1999/02/22-rdf-syntax-ns#type of all possible members")
 			.required(true)
 			.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
 			.build();
@@ -78,8 +81,11 @@ public static String getDateObservedValue(ProcessContext context) {
 		return context.getProperty(DATE_OBSERVED_VALUE_RDF_PROPERTY).getValue();
 	}
 
-	public static Resource getMemberRdfSyntaxType(ProcessContext context) {
-		return ResourceFactory.createResource(context.getProperty(MEMBER_RDF_SYNTAX_TYPE).getValue());
+	public static List<Resource> getMemberRdfSyntaxTypes(ProcessContext context) {
+		return Arrays.stream(context.getProperty(MEMBER_RDF_SYNTAX_TYPE).getValue().split(","))
+				.map(String::trim)
+				.map(ResourceFactory::createResource)
+				.toList();
 	}
 
 	public static String getDelimiter(ProcessContext context) {

ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/LdioVersionObjectCreator.java

@@ -7,13 +7,15 @@
 import org.apache.jena.rdf.model.Property;
 import org.apache.jena.rdf.model.Resource;
 
+import java.util.List;
+
 public class LdioVersionObjectCreator extends LdioTransformer {
 	public static final String NAME = "Ldio:VersionObjectCreator";
 	private final VersionObjectCreator versionObjectCreator;
 
-	public LdioVersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, Resource memberType,
+	public LdioVersionObjectCreator(PropertyExtractor dateObservedPropertyExtractor, List<Resource> memberTypes,
 	                                String delimiter, Property generatedAtProperty, Property versionOfProperty) {
-		this.versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter,
+		this.versionObjectCreator = new VersionObjectCreator(dateObservedPropertyExtractor, memberTypes, delimiter,
 				generatedAtProperty, versionOfProperty);
 	}
 

ldi-orchestrator/ldio-connectors/ldio-version-object-creator/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldio/config/LdioVersionObjectCreatorAutoConfig.java

@@ -5,6 +5,7 @@
 import be.vlaanderen.informatievlaanderen.ldes.ldi.extractor.PropertyPathExtractor;
 import be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionObjectCreator;
 import be.vlaanderen.informatievlaanderen.ldes.ldio.configurator.LdioTransformerConfigurator;
+import be.vlaanderen.informatievlaanderen.ldes.ldio.exception.ConfigPropertyMissingException;
 import be.vlaanderen.informatievlaanderen.ldes.ldio.types.LdioTransformer;
 import be.vlaanderen.informatievlaanderen.ldes.ldio.valueobjects.ComponentProperties;
 import org.apache.jena.rdf.model.Model;
@@ -14,7 +15,7 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import java.util.Optional;
+import java.util.List;
 
 import static be.vlaanderen.informatievlaanderen.ldes.ldio.LdioVersionObjectCreator.NAME;
 
@@ -27,32 +28,40 @@ public LdioTransformerConfigurator ldioConfigurator() {
 
 	public static class LdioVersionObjectCreatorTransformerConfigurator implements LdioTransformerConfigurator {
 
+		public static final String DATE_OBSERVED = "date-observed-property";
+		public static final String MEMBER_TYPE = "member-type";
+		public static final String DELIMITER = "delimiter";
+		public static final String GENERATED_AT = "generatedAt-property";
+		public static final String VERSION_OF = "versionOf-property";
 		public static final String DEFAULT_PROV_GENERATED_AT_TIME = "http://www.w3.org/ns/prov#generatedAtTime";
 		public static final String DEFAULT_VERSION_OF_KEY = "http://purl.org/dc/terms/isVersionOf";
 
 		@Override
 		public LdioTransformer configure(ComponentProperties properties) {
 			Model initModel = ModelFactory.createDefaultModel();
 
-			PropertyExtractor dateObservedPropertyExtractor = properties.getOptionalProperty("date-observed-property")
+			PropertyExtractor dateObservedPropertyExtractor = properties.getOptionalProperty(DATE_OBSERVED)
 					.map(PropertyPathExtractor::from)
 					.map(PropertyExtractor.class::cast)
 					.orElseGet(EmptyPropertyExtractor::new);
 
-			Resource memberType = Optional.of(properties.getProperty("member-type"))
-					.map(initModel::createResource).orElse(null);
+			List<Resource> memberTypes = properties.getPropertyList(MEMBER_TYPE).stream()
+					.map(initModel::createResource).toList();
+			if (memberTypes.isEmpty()) {
+				throw new ConfigPropertyMissingException(properties.getPipelineName(), properties.getComponentName(), MEMBER_TYPE);
+			}
 
-			String delimiter = properties.getOptionalProperty("delimiter").orElse("/");
+			String delimiter = properties.getOptionalProperty(DELIMITER).orElse("/");
 
-			Property generatedAtProperty = properties.getOptionalProperty("generatedAt-property")
+			Property generatedAtProperty = properties.getOptionalProperty(GENERATED_AT)
 					.map(initModel::createProperty)
 					.orElseGet(() -> initModel.createProperty(DEFAULT_PROV_GENERATED_AT_TIME));
 
-			Property versionOfProperty = properties.getOptionalProperty("versionOf-property")
+			Property versionOfProperty = properties.getOptionalProperty(VERSION_OF)
 					.map(initModel::createProperty)
 					.orElseGet(() -> initModel.createProperty(DEFAULT_VERSION_OF_KEY));
 
-			return new LdioVersionObjectCreator(dateObservedPropertyExtractor, memberType, delimiter,
+			return new LdioVersionObjectCreator(dateObservedPropertyExtractor, memberTypes, delimiter,
 					generatedAtProperty,
 					versionOfProperty);
 		}