ci: finalise release 2.6.0 (#638)

docs/_ldio/ldio-inputs/ldio-ldes-client.md

@@ -117,11 +117,17 @@ api
 | _materialisation.enable-latest-state_ | Indicates whether all state or only the latest state must be sent                     | No       | true                                 | false                                | true or false    |
 
 {: .note }
-Don't forgot to provide a timestamp-path in the general properties, as this property is not required, but necessary for this filter to work properly!
-
+Don't forgot to provide a timestamp-path in the general properties, as this property is not required, but necessary for
+this filter to work properly!
 
 {% include ldio-core/http-requester.md %}
 
+### SQLite properties
+
+| Property           | Description                                   | Required | Default | Example      | Supported values |
+|:-------------------|:----------------------------------------------|:---------|:--------|:-------------|:-----------------|
+| _sqlite.directory_ | Directory wherein the `.db` file can be saved | No       | N/A     | /ldio/sqlite | String           |
+
 ### Postgres properties
 
 | Property            | Description                                   | Required | Default | Example                                                        | Supported values |

docs/_ldio/ldio-transformers/ldio-change-detection-filter.md

@@ -0,0 +1,48 @@
+---
+layout: default
+parent: LDIO Transformers
+title: Change Detection Filter
+---
+
+# LDIO Change Detection Filter
+
+***Ldio:ChangeDetectionFilter***
+
+The LDIO Change Detection Filter, which is in fact a transformer, keeps track of each member with the same subject if
+the state has changed. If not, the member will be ignored, otherwise the member will be sent further through the
+pipeline. This can come in handy when you do not want to spam your server for example with duplicate state objects.
+
+**Flow of the Change Detection Filter**
+```mermaid
+flowchart LR
+;
+    PREV_TRANSFORMER[Previous transformer] --> State_Object((State\n object));
+    State_Object --> FILTER(Change Detection\nFilter);
+    FILTER --> HASH((Hashed\n model))
+    HASH --> Filtering{Contains\n member with\n same subject and\n same hashed\n model?};
+    Filtering -->|No| NEXT_TRANSFORMER[Next\n transformer];
+    Filtering -->|Yes| Ignore[Ignore member];
+```
+
+## Config
+
+### General properties
+
+| Property     | Description                                                                       | Required | Default | Example | Supported values                 |
+|:-------------|:----------------------------------------------------------------------------------|:---------|:--------|:--------|:---------------------------------|
+| _state_      | 'memory', 'sqlite' or 'postgres' to indicate how the state should be persisted    | No       | memory  | sqlite  | 'memory', 'sqlite' or 'postgres' |
+| _keep-state_ | Indicates if the state should be persisted on shutdown (n/a for in memory states) | No       | false   | false   | true or false                    |
+
+### SQLite properties
+
+| Property           | Description                                   | Required | Default | Example      | Supported values |
+|:-------------------|:----------------------------------------------|:---------|:--------|:-------------|:-----------------|
+| _sqlite.directory_ | Directory wherein the `.db` file can be saved | No       | N/A     | /ldio/sqlite | String           |
+
+### Postgres properties
+
+| Property            | Description                                   | Required | Default | Example                                                        | Supported values |
+|:--------------------|:----------------------------------------------|:---------|:--------|:---------------------------------------------------------------|:-----------------|
+| _postgres.url_      | JDBC URL of the Postgres database             | No       | N/A     | jdbc:postgresql://test.postgres.database.azure.com:5432/sample | String           |
+| _postgres.username_ | Username used to connect to Postgres database | No       | N/A     | myUsername@test                                                | String           |
+| _postgres.password_ | Password used to connect to Postgres database | No       | N/A     | myPassword                                                     | String           |

ldi-core/change-detection-filter/pom.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
+        <artifactId>ldi-core</artifactId>
+        <version>2.6.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>change-detection-filter</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>io.setl</groupId>
+            <artifactId>rdf-urdna</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>postgresql</artifactId>
+            <version>${testcontainers-postgresql.version}</version>
+            <scope>test</scope>
+        </dependency>
+
+
+        <dependency>
+            <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
+            <artifactId>ldi-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>be.vlaanderen.informatievlaanderen.ldes.ldi</groupId>
+            <artifactId>ldi-infra-sql</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-junit</artifactId>
+            <version>${cucumber.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-java</artifactId>
+            <version>${cucumber.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.cucumber</groupId>
+            <artifactId>cucumber-junit-platform-engine</artifactId>
+            <version>${cucumber.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.junit.platform</groupId>
+            <artifactId>junit-platform-suite</artifactId>
+            <version>${junit-platform-suite.version}</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/ChangeDetectionFilter.java

@@ -0,0 +1,124 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi;
+
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.types.LdiOneToOneTransformer;
+import com.apicatalog.jsonld.http.media.MediaType;
+import com.apicatalog.rdf.Rdf;
+import com.apicatalog.rdf.RdfDataset;
+import com.apicatalog.rdf.io.error.RdfReaderException;
+import com.apicatalog.rdf.io.error.UnsupportedContentException;
+import com.apicatalog.rdf.io.nquad.NQuadsWriter;
+import io.setl.rdf.normalization.RdfNormalize;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.ModelFactory;
+import org.apache.jena.rdf.model.RDFNode;
+import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.riot.Lang;
+import org.apache.jena.riot.RDFWriter;
+
+import java.io.*;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.List;
+
+public class ChangeDetectionFilter implements LdiOneToOneTransformer {
+	public static final String HASHING_ALGORIHTM = "SHA-256";
+	private static final Lang NORMALIZING_LANG = Lang.NQUADS;
+	private static final MediaType NORMALIZING_MEDIA_TYPE = MediaType.N_QUADS;
+	private final HashedStateMemberRepository hashedStateMemberRepository;
+	private final boolean keepState;
+
+	public ChangeDetectionFilter(HashedStateMemberRepository hashedStateMemberRepository, boolean keepState) {
+		this.hashedStateMemberRepository = hashedStateMemberRepository;
+		this.keepState = keepState;
+	}
+
+	/**
+	 * Filters out the model by returning an empty model when the model's hash has already been processed
+	 * @param model The model to be filtered
+	 * @return Either the same model if not processed yet, otherwise an empty model
+	 */
+	@Override
+	public Model transform(Model model) {
+		final Resource subject = getSingleNamedNodeFromStateObject(model);
+		final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+		canonicalizeInputModel(model, outputStream);
+		String hashedModel = hashModelBytes(outputStream.toByteArray());
+		final HashedStateMember hashedStateMember = new HashedStateMember(subject.getURI(), hashedModel);
+		if(hashedStateMemberRepository.containsHashedStateMember(hashedStateMember)) {
+			return ModelFactory.createDefaultModel();
+		}
+		hashedStateMemberRepository.saveHashedStateMember(hashedStateMember);
+		return model;
+	}
+
+	public void destroyState() {
+		if(!keepState) {
+			hashedStateMemberRepository.destroyState();
+		}
+	}
+
+
+	private Resource getSingleNamedNodeFromStateObject(Model model) {
+		final List<Resource> namedNodes = model.listSubjects().filterDrop(RDFNode::isAnon).toList();
+		if (namedNodes.size() != 1) {
+			throw new IllegalStateException("State object must contain exactly one named node");
+		}
+		return namedNodes.getFirst();
+	}
+
+	private void canonicalizeInputModel(Model model, OutputStream outputStream) {
+		final RdfDataset normalisedDataset = RdfNormalize.normalize(readDatasetFromJenaModel(model));
+		writeToOutputStream(normalisedDataset, outputStream);
+	}
+
+	private RdfDataset readDatasetFromJenaModel(Model model) {
+		final ByteArrayOutputStream receivedModelOutputStream = new ByteArrayOutputStream();
+		RDFWriter.source(model).lang(NORMALIZING_LANG).output(receivedModelOutputStream);
+		final InputStream inputStream = new ByteArrayInputStream(receivedModelOutputStream.toByteArray());
+		try {
+			return Rdf.createReader(NORMALIZING_MEDIA_TYPE, inputStream).readDataset();
+		} catch (UnsupportedContentException | IOException | RdfReaderException e) {
+			throw new IllegalStateException("Unable to read the received model", e);
+		}
+	}
+
+	private void writeToOutputStream(RdfDataset dataset, OutputStream outputStream) {
+		final Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream));
+		try {
+			new NQuadsWriter(writer).write(dataset);
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
+	}
+
+	private String hashModelBytes(byte[] modelBytes) {
+		byte[] hashedBytes = getMessageDigest().digest(modelBytes);
+		return convertHashedBytesToString(hashedBytes);
+	}
+
+	private MessageDigest getMessageDigest() {
+		try {
+			return MessageDigest.getInstance(HASHING_ALGORIHTM);
+		} catch (NoSuchAlgorithmException e) {
+			throw new IllegalArgumentException(e);
+		}
+	}
+
+	private String convertHashedBytesToString(byte[] hashedBytes) {
+		final StringBuilder hashStringBuilder = new StringBuilder();
+		for (byte b : hashedBytes) {
+			String hex = Integer.toHexString(b & 0xFF);
+			if (isLeadingZeroRequired(hex)) {
+				hashStringBuilder.append("0");
+			}
+			hashStringBuilder.append(hex);
+		}
+		return hashStringBuilder.toString();
+	}
+
+	private boolean isLeadingZeroRequired(String hex) {
+		return hex.length() == 1;
+	}
+}

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/entities/HashedStateMember.java

@@ -0,0 +1,17 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi.entities;
+
+public record HashedStateMember(String memberId, String memberHash) {
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o) return true;
+		if (!(o instanceof HashedStateMember that)) return false;
+
+		return memberId.equals(that.memberId);
+	}
+
+	@Override
+	public int hashCode() {
+		return memberId.hashCode();
+	}
+}

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/HashedStateMemberRepository.java

@@ -0,0 +1,11 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories;
+
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
+
+public interface HashedStateMemberRepository {
+	boolean containsHashedStateMember(HashedStateMember hashedStateMember);
+
+	void saveHashedStateMember(HashedStateMember hashedStateMember);
+
+	void destroyState();
+}

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/inmemory/InMemoryHashedStateMemberRepository.java

@@ -0,0 +1,30 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.inmemory;
+
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
+
+import java.util.*;
+
+public class InMemoryHashedStateMemberRepository implements HashedStateMemberRepository {
+	private final Map<String, HashedStateMember> members;
+
+	public InMemoryHashedStateMemberRepository() {
+		members = new HashMap<>();
+	}
+
+	@Override
+	public boolean containsHashedStateMember(HashedStateMember hashedStateMember) {
+		final HashedStateMember member = members.get(hashedStateMember.memberId());
+		return member != null && member.memberHash().equals(hashedStateMember.memberHash());
+	}
+
+	@Override
+	public void saveHashedStateMember(HashedStateMember hashedStateMember) {
+		members.put(hashedStateMember.memberId(), hashedStateMember);
+	}
+
+	@Override
+	public void destroyState() {
+		members.clear();
+	}
+}

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/mapper/HashedStateMemberEntityMapper.java

@@ -0,0 +1,13 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.mapper;
+
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMemberEntity;
+
+public class HashedStateMemberEntityMapper {
+	private HashedStateMemberEntityMapper() {
+	}
+
+	public static HashedStateMemberEntity fromHashedStateMember(HashedStateMember hashedStateMember) {
+		return new HashedStateMemberEntity(hashedStateMember.memberId(), hashedStateMember.memberHash());
+	}
+}

ldi-core/change-detection-filter/src/main/java/be/vlaanderen/informatievlaanderen/ldes/ldi/repositories/sql/SqlHashedStateMemberRepository.java

@@ -0,0 +1,44 @@
+package be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.sql;
+
+import be.vlaanderen.informatievlaanderen.ldes.ldi.EntityManagerFactory;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMember;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.entities.HashedStateMemberEntity;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.HashedStateMemberRepository;
+import be.vlaanderen.informatievlaanderen.ldes.ldi.repositories.mapper.HashedStateMemberEntityMapper;
+
+import javax.persistence.EntityManager;
+
+public class SqlHashedStateMemberRepository implements HashedStateMemberRepository {
+	private final EntityManagerFactory entityManagerFactory;
+	private final EntityManager entityManager;
+	private final String instanceName;
+
+	public SqlHashedStateMemberRepository(EntityManagerFactory entityManagerFactory, String instanceName) {
+		this.entityManagerFactory = entityManagerFactory;
+		this.entityManager = entityManagerFactory.getEntityManager();
+		this.instanceName = instanceName;
+	}
+
+	@Override
+	public boolean containsHashedStateMember(HashedStateMember hashedStateMember) {
+		return entityManager
+				.createNamedQuery("HashedStateMember.findMember", HashedStateMemberEntity.class)
+				.setParameter("memberId", hashedStateMember.memberId())
+				.setParameter("memberHash", hashedStateMember.memberHash())
+				.getResultStream()
+				.findFirst()
+				.isPresent();
+	}
+
+	@Override
+	public void saveHashedStateMember(HashedStateMember hashedStateMember) {
+		entityManager.getTransaction().begin();
+		entityManager.merge(HashedStateMemberEntityMapper.fromHashedStateMember(hashedStateMember));
+		entityManager.getTransaction().commit();
+	}
+
+	@Override
+	public void destroyState() {
+		entityManagerFactory.destroyState(instanceName);
+	}
+}